1 /*
2  * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <net/tc_act/tc_gact.h>
34 #include <net/pkt_cls.h>
35 #include <linux/mlx5/fs.h>
36 #include <net/vxlan.h>
37 #include <linux/bpf.h>
38 #include <net/page_pool.h>
39 #include "eswitch.h"
40 #include "en.h"
41 #include "en_tc.h"
42 #include "en_rep.h"
43 #include "en_accel/ipsec.h"
44 #include "en_accel/ipsec_rxtx.h"
45 #include "en_accel/tls.h"
46 #include "accel/ipsec.h"
47 #include "accel/tls.h"
48 #include "lib/vxlan.h"
49 #include "lib/clock.h"
50 #include "en/port.h"
51 #include "en/xdp.h"
52 
53 struct mlx5e_rq_param {
54 	u32			rqc[MLX5_ST_SZ_DW(rqc)];
55 	struct mlx5_wq_param	wq;
56 	struct mlx5e_rq_frags_info frags_info;
57 };
58 
59 struct mlx5e_sq_param {
60 	u32                        sqc[MLX5_ST_SZ_DW(sqc)];
61 	struct mlx5_wq_param       wq;
62 };
63 
64 struct mlx5e_cq_param {
65 	u32                        cqc[MLX5_ST_SZ_DW(cqc)];
66 	struct mlx5_wq_param       wq;
67 	u16                        eq_ix;
68 	u8                         cq_period_mode;
69 };
70 
71 struct mlx5e_channel_param {
72 	struct mlx5e_rq_param      rq;
73 	struct mlx5e_sq_param      sq;
74 	struct mlx5e_sq_param      xdp_sq;
75 	struct mlx5e_sq_param      icosq;
76 	struct mlx5e_cq_param      rx_cq;
77 	struct mlx5e_cq_param      tx_cq;
78 	struct mlx5e_cq_param      icosq_cq;
79 };
80 
mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev * mdev)81 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev)
82 {
83 	bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) &&
84 		MLX5_CAP_GEN(mdev, umr_ptr_rlky) &&
85 		MLX5_CAP_ETH(mdev, reg_umr_sq);
86 	u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq);
87 	bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap;
88 
89 	if (!striding_rq_umr)
90 		return false;
91 	if (!inline_umr) {
92 		mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n",
93 			       (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap);
94 		return false;
95 	}
96 	return true;
97 }
98 
mlx5e_rx_get_linear_frag_sz(struct mlx5e_params * params)99 static u32 mlx5e_rx_get_linear_frag_sz(struct mlx5e_params *params)
100 {
101 	u16 hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu);
102 	u16 linear_rq_headroom = params->xdp_prog ?
103 		XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
104 	u32 frag_sz;
105 
106 	linear_rq_headroom += NET_IP_ALIGN;
107 
108 	frag_sz = MLX5_SKB_FRAG_SZ(linear_rq_headroom + hw_mtu);
109 
110 	if (params->xdp_prog && frag_sz < PAGE_SIZE)
111 		frag_sz = PAGE_SIZE;
112 
113 	return frag_sz;
114 }
115 
mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params * params)116 static u8 mlx5e_mpwqe_log_pkts_per_wqe(struct mlx5e_params *params)
117 {
118 	u32 linear_frag_sz = mlx5e_rx_get_linear_frag_sz(params);
119 
120 	return MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(linear_frag_sz);
121 }
122 
mlx5e_rx_is_linear_skb(struct mlx5_core_dev * mdev,struct mlx5e_params * params)123 static bool mlx5e_rx_is_linear_skb(struct mlx5_core_dev *mdev,
124 				   struct mlx5e_params *params)
125 {
126 	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
127 
128 	return !params->lro_en && frag_sz <= PAGE_SIZE;
129 }
130 
mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev * mdev,struct mlx5e_params * params)131 static bool mlx5e_rx_mpwqe_is_linear_skb(struct mlx5_core_dev *mdev,
132 					 struct mlx5e_params *params)
133 {
134 	u32 frag_sz = mlx5e_rx_get_linear_frag_sz(params);
135 	s8 signed_log_num_strides_param;
136 	u8 log_num_strides;
137 
138 	if (!mlx5e_rx_is_linear_skb(mdev, params))
139 		return false;
140 
141 	if (MLX5_CAP_GEN(mdev, ext_stride_num_range))
142 		return true;
143 
144 	log_num_strides = MLX5_MPWRQ_LOG_WQE_SZ - order_base_2(frag_sz);
145 	signed_log_num_strides_param =
146 		(s8)log_num_strides - MLX5_MPWQE_LOG_NUM_STRIDES_BASE;
147 
148 	return signed_log_num_strides_param >= 0;
149 }
150 
mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params * params)151 static u8 mlx5e_mpwqe_get_log_rq_size(struct mlx5e_params *params)
152 {
153 	if (params->log_rq_mtu_frames <
154 	    mlx5e_mpwqe_log_pkts_per_wqe(params) + MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW)
155 		return MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW;
156 
157 	return params->log_rq_mtu_frames - mlx5e_mpwqe_log_pkts_per_wqe(params);
158 }
159 
mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev * mdev,struct mlx5e_params * params)160 static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
161 					  struct mlx5e_params *params)
162 {
163 	if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
164 		return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
165 
166 	return MLX5E_MPWQE_STRIDE_SZ(mdev,
167 		MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
168 }
169 
mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev * mdev,struct mlx5e_params * params)170 static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
171 					  struct mlx5e_params *params)
172 {
173 	return MLX5_MPWRQ_LOG_WQE_SZ -
174 		mlx5e_mpwqe_get_log_stride_size(mdev, params);
175 }
176 
mlx5e_get_rq_headroom(struct mlx5_core_dev * mdev,struct mlx5e_params * params)177 static u16 mlx5e_get_rq_headroom(struct mlx5_core_dev *mdev,
178 				 struct mlx5e_params *params)
179 {
180 	u16 linear_rq_headroom = params->xdp_prog ?
181 		XDP_PACKET_HEADROOM : MLX5_RX_HEADROOM;
182 	bool is_linear_skb;
183 
184 	linear_rq_headroom += NET_IP_ALIGN;
185 
186 	is_linear_skb = (params->rq_wq_type == MLX5_WQ_TYPE_CYCLIC) ?
187 		mlx5e_rx_is_linear_skb(mdev, params) :
188 		mlx5e_rx_mpwqe_is_linear_skb(mdev, params);
189 
190 	return is_linear_skb ? linear_rq_headroom : 0;
191 }
192 
mlx5e_init_rq_type_params(struct mlx5_core_dev * mdev,struct mlx5e_params * params)193 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev,
194 			       struct mlx5e_params *params)
195 {
196 	params->lro_wqe_sz = MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ;
197 	params->log_rq_mtu_frames = is_kdump_kernel() ?
198 		MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE :
199 		MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE;
200 
201 	mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n",
202 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ,
203 		       params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ?
204 		       BIT(mlx5e_mpwqe_get_log_rq_size(params)) :
205 		       BIT(params->log_rq_mtu_frames),
206 		       BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)),
207 		       MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
208 }
209 
mlx5e_striding_rq_possible(struct mlx5_core_dev * mdev,struct mlx5e_params * params)210 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev,
211 				struct mlx5e_params *params)
212 {
213 	return mlx5e_check_fragmented_striding_rq_cap(mdev) &&
214 		!MLX5_IPSEC_DEV(mdev) &&
215 		!(params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params));
216 }
217 
mlx5e_set_rq_type(struct mlx5_core_dev * mdev,struct mlx5e_params * params)218 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params)
219 {
220 	params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) &&
221 		MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ?
222 		MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ :
223 		MLX5_WQ_TYPE_CYCLIC;
224 }
225 
mlx5e_update_carrier(struct mlx5e_priv * priv)226 static void mlx5e_update_carrier(struct mlx5e_priv *priv)
227 {
228 	struct mlx5_core_dev *mdev = priv->mdev;
229 	u8 port_state;
230 
231 	port_state = mlx5_query_vport_state(mdev,
232 					    MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT,
233 					    0);
234 
235 	if (port_state == VPORT_STATE_UP) {
236 		netdev_info(priv->netdev, "Link up\n");
237 		netif_carrier_on(priv->netdev);
238 	} else {
239 		netdev_info(priv->netdev, "Link down\n");
240 		netif_carrier_off(priv->netdev);
241 	}
242 }
243 
mlx5e_update_carrier_work(struct work_struct * work)244 static void mlx5e_update_carrier_work(struct work_struct *work)
245 {
246 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
247 					       update_carrier_work);
248 
249 	mutex_lock(&priv->state_lock);
250 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
251 		if (priv->profile->update_carrier)
252 			priv->profile->update_carrier(priv);
253 	mutex_unlock(&priv->state_lock);
254 }
255 
mlx5e_update_stats(struct mlx5e_priv * priv)256 void mlx5e_update_stats(struct mlx5e_priv *priv)
257 {
258 	int i;
259 
260 	for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
261 		if (mlx5e_stats_grps[i].update_stats)
262 			mlx5e_stats_grps[i].update_stats(priv);
263 }
264 
mlx5e_update_ndo_stats(struct mlx5e_priv * priv)265 static void mlx5e_update_ndo_stats(struct mlx5e_priv *priv)
266 {
267 	int i;
268 
269 	for (i = mlx5e_num_stats_grps - 1; i >= 0; i--)
270 		if (mlx5e_stats_grps[i].update_stats_mask &
271 		    MLX5E_NDO_UPDATE_STATS)
272 			mlx5e_stats_grps[i].update_stats(priv);
273 }
274 
mlx5e_update_stats_work(struct work_struct * work)275 void mlx5e_update_stats_work(struct work_struct *work)
276 {
277 	struct delayed_work *dwork = to_delayed_work(work);
278 	struct mlx5e_priv *priv = container_of(dwork, struct mlx5e_priv,
279 					       update_stats_work);
280 
281 	mutex_lock(&priv->state_lock);
282 	priv->profile->update_stats(priv);
283 	mutex_unlock(&priv->state_lock);
284 }
285 
mlx5e_async_event(struct mlx5_core_dev * mdev,void * vpriv,enum mlx5_dev_event event,unsigned long param)286 static void mlx5e_async_event(struct mlx5_core_dev *mdev, void *vpriv,
287 			      enum mlx5_dev_event event, unsigned long param)
288 {
289 	struct mlx5e_priv *priv = vpriv;
290 
291 	if (!test_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state))
292 		return;
293 
294 	switch (event) {
295 	case MLX5_DEV_EVENT_PORT_UP:
296 	case MLX5_DEV_EVENT_PORT_DOWN:
297 		queue_work(priv->wq, &priv->update_carrier_work);
298 		break;
299 	default:
300 		break;
301 	}
302 }
303 
mlx5e_enable_async_events(struct mlx5e_priv * priv)304 static void mlx5e_enable_async_events(struct mlx5e_priv *priv)
305 {
306 	set_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
307 }
308 
mlx5e_disable_async_events(struct mlx5e_priv * priv)309 static void mlx5e_disable_async_events(struct mlx5e_priv *priv)
310 {
311 	clear_bit(MLX5E_STATE_ASYNC_EVENTS_ENABLED, &priv->state);
312 	synchronize_irq(pci_irq_vector(priv->mdev->pdev, MLX5_EQ_VEC_ASYNC));
313 }
314 
mlx5e_build_umr_wqe(struct mlx5e_rq * rq,struct mlx5e_icosq * sq,struct mlx5e_umr_wqe * wqe)315 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq,
316 				       struct mlx5e_icosq *sq,
317 				       struct mlx5e_umr_wqe *wqe)
318 {
319 	struct mlx5_wqe_ctrl_seg      *cseg = &wqe->ctrl;
320 	struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl;
321 	u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS);
322 
323 	cseg->qpn_ds    = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) |
324 				      ds_cnt);
325 	cseg->fm_ce_se  = MLX5_WQE_CTRL_CQ_UPDATE;
326 	cseg->imm       = rq->mkey_be;
327 
328 	ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE;
329 	ucseg->xlt_octowords =
330 		cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE));
331 	ucseg->mkey_mask     = cpu_to_be64(MLX5_MKEY_MASK_FREE);
332 }
333 
mlx5e_rqwq_get_size(struct mlx5e_rq * rq)334 static u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq)
335 {
336 	switch (rq->wq_type) {
337 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
338 		return mlx5_wq_ll_get_size(&rq->mpwqe.wq);
339 	default:
340 		return mlx5_wq_cyc_get_size(&rq->wqe.wq);
341 	}
342 }
343 
mlx5e_rqwq_get_cur_sz(struct mlx5e_rq * rq)344 static u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq)
345 {
346 	switch (rq->wq_type) {
347 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
348 		return rq->mpwqe.wq.cur_sz;
349 	default:
350 		return rq->wqe.wq.cur_sz;
351 	}
352 }
353 
mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq * rq,struct mlx5e_channel * c)354 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq,
355 				     struct mlx5e_channel *c)
356 {
357 	int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
358 
359 	rq->mpwqe.info = kvzalloc_node(array_size(wq_sz,
360 						  sizeof(*rq->mpwqe.info)),
361 				       GFP_KERNEL, cpu_to_node(c->cpu));
362 	if (!rq->mpwqe.info)
363 		return -ENOMEM;
364 
365 	mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe);
366 
367 	return 0;
368 }
369 
mlx5e_create_umr_mkey(struct mlx5_core_dev * mdev,u64 npages,u8 page_shift,struct mlx5_core_mkey * umr_mkey)370 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev,
371 				 u64 npages, u8 page_shift,
372 				 struct mlx5_core_mkey *umr_mkey)
373 {
374 	int inlen = MLX5_ST_SZ_BYTES(create_mkey_in);
375 	void *mkc;
376 	u32 *in;
377 	int err;
378 
379 	in = kvzalloc(inlen, GFP_KERNEL);
380 	if (!in)
381 		return -ENOMEM;
382 
383 	mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
384 
385 	MLX5_SET(mkc, mkc, free, 1);
386 	MLX5_SET(mkc, mkc, umr_en, 1);
387 	MLX5_SET(mkc, mkc, lw, 1);
388 	MLX5_SET(mkc, mkc, lr, 1);
389 	MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
390 
391 	MLX5_SET(mkc, mkc, qpn, 0xffffff);
392 	MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn);
393 	MLX5_SET64(mkc, mkc, len, npages << page_shift);
394 	MLX5_SET(mkc, mkc, translations_octword_size,
395 		 MLX5_MTT_OCTW(npages));
396 	MLX5_SET(mkc, mkc, log_page_size, page_shift);
397 
398 	err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen);
399 
400 	kvfree(in);
401 	return err;
402 }
403 
mlx5e_create_rq_umr_mkey(struct mlx5_core_dev * mdev,struct mlx5e_rq * rq)404 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq)
405 {
406 	u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq));
407 
408 	return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey);
409 }
410 
mlx5e_get_mpwqe_offset(struct mlx5e_rq * rq,u16 wqe_ix)411 static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix)
412 {
413 	return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT;
414 }
415 
mlx5e_init_frags_partition(struct mlx5e_rq * rq)416 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq)
417 {
418 	struct mlx5e_wqe_frag_info next_frag, *prev;
419 	int i;
420 
421 	next_frag.di = &rq->wqe.di[0];
422 	next_frag.offset = 0;
423 	prev = NULL;
424 
425 	for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) {
426 		struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0];
427 		struct mlx5e_wqe_frag_info *frag =
428 			&rq->wqe.frags[i << rq->wqe.info.log_num_frags];
429 		int f;
430 
431 		for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) {
432 			if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) {
433 				next_frag.di++;
434 				next_frag.offset = 0;
435 				if (prev)
436 					prev->last_in_page = true;
437 			}
438 			*frag = next_frag;
439 
440 			/* prepare next */
441 			next_frag.offset += frag_info[f].frag_stride;
442 			prev = frag;
443 		}
444 	}
445 
446 	if (prev)
447 		prev->last_in_page = true;
448 }
449 
mlx5e_init_di_list(struct mlx5e_rq * rq,struct mlx5e_params * params,int wq_sz,int cpu)450 static int mlx5e_init_di_list(struct mlx5e_rq *rq,
451 			      struct mlx5e_params *params,
452 			      int wq_sz, int cpu)
453 {
454 	int len = wq_sz << rq->wqe.info.log_num_frags;
455 
456 	rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)),
457 				   GFP_KERNEL, cpu_to_node(cpu));
458 	if (!rq->wqe.di)
459 		return -ENOMEM;
460 
461 	mlx5e_init_frags_partition(rq);
462 
463 	return 0;
464 }
465 
mlx5e_free_di_list(struct mlx5e_rq * rq)466 static void mlx5e_free_di_list(struct mlx5e_rq *rq)
467 {
468 	kvfree(rq->wqe.di);
469 }
470 
mlx5e_alloc_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_rq_param * rqp,struct mlx5e_rq * rq)471 static int mlx5e_alloc_rq(struct mlx5e_channel *c,
472 			  struct mlx5e_params *params,
473 			  struct mlx5e_rq_param *rqp,
474 			  struct mlx5e_rq *rq)
475 {
476 	struct page_pool_params pp_params = { 0 };
477 	struct mlx5_core_dev *mdev = c->mdev;
478 	void *rqc = rqp->rqc;
479 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
480 	u32 pool_size;
481 	int wq_sz;
482 	int err;
483 	int i;
484 
485 	rqp->wq.db_numa_node = cpu_to_node(c->cpu);
486 
487 	rq->wq_type = params->rq_wq_type;
488 	rq->pdev    = c->pdev;
489 	rq->netdev  = c->netdev;
490 	rq->tstamp  = c->tstamp;
491 	rq->clock   = &mdev->clock;
492 	rq->channel = c;
493 	rq->ix      = c->ix;
494 	rq->mdev    = mdev;
495 	rq->stats   = &c->priv->channel_stats[c->ix].rq;
496 
497 	rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL;
498 	if (IS_ERR(rq->xdp_prog)) {
499 		err = PTR_ERR(rq->xdp_prog);
500 		rq->xdp_prog = NULL;
501 		goto err_rq_wq_destroy;
502 	}
503 
504 	err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix);
505 	if (err < 0)
506 		goto err_rq_wq_destroy;
507 
508 	rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE;
509 	rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params);
510 	pool_size = 1 << params->log_rq_mtu_frames;
511 
512 	switch (rq->wq_type) {
513 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
514 		err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq,
515 					&rq->wq_ctrl);
516 		if (err)
517 			return err;
518 
519 		rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR];
520 
521 		wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq);
522 
523 		pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params);
524 
525 		rq->post_wqes = mlx5e_post_rx_mpwqes;
526 		rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe;
527 
528 		rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe;
529 #ifdef CONFIG_MLX5_EN_IPSEC
530 		if (MLX5_IPSEC_DEV(mdev)) {
531 			err = -EINVAL;
532 			netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n");
533 			goto err_rq_wq_destroy;
534 		}
535 #endif
536 		if (!rq->handle_rx_cqe) {
537 			err = -EINVAL;
538 			netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err);
539 			goto err_rq_wq_destroy;
540 		}
541 
542 		rq->mpwqe.skb_from_cqe_mpwrq =
543 			mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ?
544 			mlx5e_skb_from_cqe_mpwrq_linear :
545 			mlx5e_skb_from_cqe_mpwrq_nonlinear;
546 		rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params);
547 		rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params));
548 
549 		err = mlx5e_create_rq_umr_mkey(mdev, rq);
550 		if (err)
551 			goto err_rq_wq_destroy;
552 		rq->mkey_be = cpu_to_be32(rq->umr_mkey.key);
553 
554 		err = mlx5e_rq_alloc_mpwqe_info(rq, c);
555 		if (err)
556 			goto err_free;
557 		break;
558 	default: /* MLX5_WQ_TYPE_CYCLIC */
559 		err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq,
560 					 &rq->wq_ctrl);
561 		if (err)
562 			return err;
563 
564 		rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR];
565 
566 		wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq);
567 
568 		rq->wqe.info = rqp->frags_info;
569 		rq->wqe.frags =
570 			kvzalloc_node(array_size(sizeof(*rq->wqe.frags),
571 					(wq_sz << rq->wqe.info.log_num_frags)),
572 				      GFP_KERNEL, cpu_to_node(c->cpu));
573 		if (!rq->wqe.frags) {
574 			err = -ENOMEM;
575 			goto err_free;
576 		}
577 
578 		err = mlx5e_init_di_list(rq, params, wq_sz, c->cpu);
579 		if (err)
580 			goto err_free;
581 		rq->post_wqes = mlx5e_post_rx_wqes;
582 		rq->dealloc_wqe = mlx5e_dealloc_rx_wqe;
583 
584 #ifdef CONFIG_MLX5_EN_IPSEC
585 		if (c->priv->ipsec)
586 			rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe;
587 		else
588 #endif
589 			rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe;
590 		if (!rq->handle_rx_cqe) {
591 			err = -EINVAL;
592 			netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err);
593 			goto err_free;
594 		}
595 
596 		rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(mdev, params) ?
597 			mlx5e_skb_from_cqe_linear :
598 			mlx5e_skb_from_cqe_nonlinear;
599 		rq->mkey_be = c->mkey_be;
600 	}
601 
602 	/* Create a page_pool and register it with rxq */
603 	pp_params.order     = 0;
604 	pp_params.flags     = 0; /* No-internal DMA mapping in page_pool */
605 	pp_params.pool_size = pool_size;
606 	pp_params.nid       = cpu_to_node(c->cpu);
607 	pp_params.dev       = c->pdev;
608 	pp_params.dma_dir   = rq->buff.map_dir;
609 
610 	/* page_pool can be used even when there is no rq->xdp_prog,
611 	 * given page_pool does not handle DMA mapping there is no
612 	 * required state to clear. And page_pool gracefully handle
613 	 * elevated refcnt.
614 	 */
615 	rq->page_pool = page_pool_create(&pp_params);
616 	if (IS_ERR(rq->page_pool)) {
617 		err = PTR_ERR(rq->page_pool);
618 		rq->page_pool = NULL;
619 		goto err_free;
620 	}
621 	err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq,
622 					 MEM_TYPE_PAGE_POOL, rq->page_pool);
623 	if (err)
624 		goto err_free;
625 
626 	for (i = 0; i < wq_sz; i++) {
627 		if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
628 			struct mlx5e_rx_wqe_ll *wqe =
629 				mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i);
630 			u32 byte_count =
631 				rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz;
632 			u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i);
633 
634 			wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom);
635 			wqe->data[0].byte_count = cpu_to_be32(byte_count);
636 			wqe->data[0].lkey = rq->mkey_be;
637 		} else {
638 			struct mlx5e_rx_wqe_cyc *wqe =
639 				mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i);
640 			int f;
641 
642 			for (f = 0; f < rq->wqe.info.num_frags; f++) {
643 				u32 frag_size = rq->wqe.info.arr[f].frag_size |
644 					MLX5_HW_START_PADDING;
645 
646 				wqe->data[f].byte_count = cpu_to_be32(frag_size);
647 				wqe->data[f].lkey = rq->mkey_be;
648 			}
649 			/* check if num_frags is not a pow of two */
650 			if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) {
651 				wqe->data[f].byte_count = 0;
652 				wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY);
653 				wqe->data[f].addr = 0;
654 			}
655 		}
656 	}
657 
658 	INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work);
659 
660 	switch (params->rx_cq_moderation.cq_period_mode) {
661 	case MLX5_CQ_PERIOD_MODE_START_FROM_CQE:
662 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE;
663 		break;
664 	case MLX5_CQ_PERIOD_MODE_START_FROM_EQE:
665 	default:
666 		rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
667 	}
668 
669 	rq->page_cache.head = 0;
670 	rq->page_cache.tail = 0;
671 
672 	return 0;
673 
674 err_free:
675 	switch (rq->wq_type) {
676 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
677 		kvfree(rq->mpwqe.info);
678 		mlx5_core_destroy_mkey(mdev, &rq->umr_mkey);
679 		break;
680 	default: /* MLX5_WQ_TYPE_CYCLIC */
681 		kvfree(rq->wqe.frags);
682 		mlx5e_free_di_list(rq);
683 	}
684 
685 err_rq_wq_destroy:
686 	if (rq->xdp_prog)
687 		bpf_prog_put(rq->xdp_prog);
688 	xdp_rxq_info_unreg(&rq->xdp_rxq);
689 	if (rq->page_pool)
690 		page_pool_destroy(rq->page_pool);
691 	mlx5_wq_destroy(&rq->wq_ctrl);
692 
693 	return err;
694 }
695 
mlx5e_free_rq(struct mlx5e_rq * rq)696 static void mlx5e_free_rq(struct mlx5e_rq *rq)
697 {
698 	int i;
699 
700 	if (rq->xdp_prog)
701 		bpf_prog_put(rq->xdp_prog);
702 
703 	xdp_rxq_info_unreg(&rq->xdp_rxq);
704 	if (rq->page_pool)
705 		page_pool_destroy(rq->page_pool);
706 
707 	switch (rq->wq_type) {
708 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
709 		kvfree(rq->mpwqe.info);
710 		mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey);
711 		break;
712 	default: /* MLX5_WQ_TYPE_CYCLIC */
713 		kvfree(rq->wqe.frags);
714 		mlx5e_free_di_list(rq);
715 	}
716 
717 	for (i = rq->page_cache.head; i != rq->page_cache.tail;
718 	     i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) {
719 		struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i];
720 
721 		mlx5e_page_release(rq, dma_info, false);
722 	}
723 	mlx5_wq_destroy(&rq->wq_ctrl);
724 }
725 
mlx5e_create_rq(struct mlx5e_rq * rq,struct mlx5e_rq_param * param)726 static int mlx5e_create_rq(struct mlx5e_rq *rq,
727 			   struct mlx5e_rq_param *param)
728 {
729 	struct mlx5_core_dev *mdev = rq->mdev;
730 
731 	void *in;
732 	void *rqc;
733 	void *wq;
734 	int inlen;
735 	int err;
736 
737 	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
738 		sizeof(u64) * rq->wq_ctrl.buf.npages;
739 	in = kvzalloc(inlen, GFP_KERNEL);
740 	if (!in)
741 		return -ENOMEM;
742 
743 	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
744 	wq  = MLX5_ADDR_OF(rqc, rqc, wq);
745 
746 	memcpy(rqc, param->rqc, sizeof(param->rqc));
747 
748 	MLX5_SET(rqc,  rqc, cqn,		rq->cq.mcq.cqn);
749 	MLX5_SET(rqc,  rqc, state,		MLX5_RQC_STATE_RST);
750 	MLX5_SET(wq,   wq,  log_wq_pg_sz,	rq->wq_ctrl.buf.page_shift -
751 						MLX5_ADAPTER_PAGE_SHIFT);
752 	MLX5_SET64(wq, wq,  dbr_addr,		rq->wq_ctrl.db.dma);
753 
754 	mlx5_fill_page_frag_array(&rq->wq_ctrl.buf,
755 				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
756 
757 	err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn);
758 
759 	kvfree(in);
760 
761 	return err;
762 }
763 
mlx5e_modify_rq_state(struct mlx5e_rq * rq,int curr_state,int next_state)764 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state,
765 				 int next_state)
766 {
767 	struct mlx5_core_dev *mdev = rq->mdev;
768 
769 	void *in;
770 	void *rqc;
771 	int inlen;
772 	int err;
773 
774 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
775 	in = kvzalloc(inlen, GFP_KERNEL);
776 	if (!in)
777 		return -ENOMEM;
778 
779 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
780 
781 	MLX5_SET(modify_rq_in, in, rq_state, curr_state);
782 	MLX5_SET(rqc, rqc, state, next_state);
783 
784 	err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
785 
786 	kvfree(in);
787 
788 	return err;
789 }
790 
mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq * rq,bool enable)791 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable)
792 {
793 	struct mlx5e_channel *c = rq->channel;
794 	struct mlx5e_priv *priv = c->priv;
795 	struct mlx5_core_dev *mdev = priv->mdev;
796 
797 	void *in;
798 	void *rqc;
799 	int inlen;
800 	int err;
801 
802 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
803 	in = kvzalloc(inlen, GFP_KERNEL);
804 	if (!in)
805 		return -ENOMEM;
806 
807 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
808 
809 	MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
810 	MLX5_SET64(modify_rq_in, in, modify_bitmask,
811 		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS);
812 	MLX5_SET(rqc, rqc, scatter_fcs, enable);
813 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
814 
815 	err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
816 
817 	kvfree(in);
818 
819 	return err;
820 }
821 
mlx5e_modify_rq_vsd(struct mlx5e_rq * rq,bool vsd)822 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd)
823 {
824 	struct mlx5e_channel *c = rq->channel;
825 	struct mlx5_core_dev *mdev = c->mdev;
826 	void *in;
827 	void *rqc;
828 	int inlen;
829 	int err;
830 
831 	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
832 	in = kvzalloc(inlen, GFP_KERNEL);
833 	if (!in)
834 		return -ENOMEM;
835 
836 	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
837 
838 	MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY);
839 	MLX5_SET64(modify_rq_in, in, modify_bitmask,
840 		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
841 	MLX5_SET(rqc, rqc, vsd, vsd);
842 	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
843 
844 	err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen);
845 
846 	kvfree(in);
847 
848 	return err;
849 }
850 
mlx5e_destroy_rq(struct mlx5e_rq * rq)851 static void mlx5e_destroy_rq(struct mlx5e_rq *rq)
852 {
853 	mlx5_core_destroy_rq(rq->mdev, rq->rqn);
854 }
855 
mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq * rq,int wait_time)856 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time)
857 {
858 	unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time);
859 	struct mlx5e_channel *c = rq->channel;
860 
861 	u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq));
862 
863 	do {
864 		if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes)
865 			return 0;
866 
867 		msleep(20);
868 	} while (time_before(jiffies, exp_time));
869 
870 	netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n",
871 		    c->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes);
872 
873 	return -ETIMEDOUT;
874 }
875 
mlx5e_free_rx_descs(struct mlx5e_rq * rq)876 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq)
877 {
878 	__be16 wqe_ix_be;
879 	u16 wqe_ix;
880 
881 	if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
882 		struct mlx5_wq_ll *wq = &rq->mpwqe.wq;
883 
884 		/* UMR WQE (if in progress) is always at wq->head */
885 		if (rq->mpwqe.umr_in_progress)
886 			rq->dealloc_wqe(rq, wq->head);
887 
888 		while (!mlx5_wq_ll_is_empty(wq)) {
889 			struct mlx5e_rx_wqe_ll *wqe;
890 
891 			wqe_ix_be = *wq->tail_next;
892 			wqe_ix    = be16_to_cpu(wqe_ix_be);
893 			wqe       = mlx5_wq_ll_get_wqe(wq, wqe_ix);
894 			rq->dealloc_wqe(rq, wqe_ix);
895 			mlx5_wq_ll_pop(wq, wqe_ix_be,
896 				       &wqe->next.next_wqe_index);
897 		}
898 	} else {
899 		struct mlx5_wq_cyc *wq = &rq->wqe.wq;
900 
901 		while (!mlx5_wq_cyc_is_empty(wq)) {
902 			wqe_ix = mlx5_wq_cyc_get_tail(wq);
903 			rq->dealloc_wqe(rq, wqe_ix);
904 			mlx5_wq_cyc_pop(wq);
905 		}
906 	}
907 
908 }
909 
mlx5e_open_rq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_rq_param * param,struct mlx5e_rq * rq)910 static int mlx5e_open_rq(struct mlx5e_channel *c,
911 			 struct mlx5e_params *params,
912 			 struct mlx5e_rq_param *param,
913 			 struct mlx5e_rq *rq)
914 {
915 	int err;
916 
917 	err = mlx5e_alloc_rq(c, params, param, rq);
918 	if (err)
919 		return err;
920 
921 	err = mlx5e_create_rq(rq, param);
922 	if (err)
923 		goto err_free_rq;
924 
925 	err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
926 	if (err)
927 		goto err_destroy_rq;
928 
929 	if (params->rx_dim_enabled)
930 		__set_bit(MLX5E_RQ_STATE_AM, &c->rq.state);
931 
932 	return 0;
933 
934 err_destroy_rq:
935 	mlx5e_destroy_rq(rq);
936 err_free_rq:
937 	mlx5e_free_rq(rq);
938 
939 	return err;
940 }
941 
mlx5e_activate_rq(struct mlx5e_rq * rq)942 static void mlx5e_activate_rq(struct mlx5e_rq *rq)
943 {
944 	struct mlx5e_icosq *sq = &rq->channel->icosq;
945 	struct mlx5_wq_cyc *wq = &sq->wq;
946 	struct mlx5e_tx_wqe *nopwqe;
947 
948 	u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
949 
950 	set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
951 	sq->db.ico_wqe[pi].opcode     = MLX5_OPCODE_NOP;
952 	nopwqe = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
953 	mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nopwqe->ctrl);
954 }
955 
mlx5e_deactivate_rq(struct mlx5e_rq * rq)956 static void mlx5e_deactivate_rq(struct mlx5e_rq *rq)
957 {
958 	clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state);
959 	napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */
960 }
961 
mlx5e_close_rq(struct mlx5e_rq * rq)962 static void mlx5e_close_rq(struct mlx5e_rq *rq)
963 {
964 	cancel_work_sync(&rq->dim.work);
965 	mlx5e_destroy_rq(rq);
966 	mlx5e_free_rx_descs(rq);
967 	mlx5e_free_rq(rq);
968 }
969 
mlx5e_free_xdpsq_db(struct mlx5e_xdpsq * sq)970 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq)
971 {
972 	kvfree(sq->db.xdpi);
973 }
974 
mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq * sq,int numa)975 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa)
976 {
977 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
978 
979 	sq->db.xdpi = kvzalloc_node(array_size(wq_sz, sizeof(*sq->db.xdpi)),
980 				    GFP_KERNEL, numa);
981 	if (!sq->db.xdpi) {
982 		mlx5e_free_xdpsq_db(sq);
983 		return -ENOMEM;
984 	}
985 
986 	return 0;
987 }
988 
mlx5e_alloc_xdpsq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_xdpsq * sq,bool is_redirect)989 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c,
990 			     struct mlx5e_params *params,
991 			     struct mlx5e_sq_param *param,
992 			     struct mlx5e_xdpsq *sq,
993 			     bool is_redirect)
994 {
995 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
996 	struct mlx5_core_dev *mdev = c->mdev;
997 	struct mlx5_wq_cyc *wq = &sq->wq;
998 	int err;
999 
1000 	sq->pdev      = c->pdev;
1001 	sq->mkey_be   = c->mkey_be;
1002 	sq->channel   = c;
1003 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1004 	sq->min_inline_mode = params->tx_min_inline_mode;
1005 	sq->hw_mtu    = MLX5E_SW2HW_MTU(params, params->sw_mtu);
1006 	sq->stats     = is_redirect ?
1007 		&c->priv->channel_stats[c->ix].xdpsq :
1008 		&c->priv->channel_stats[c->ix].rq_xdpsq;
1009 
1010 	param->wq.db_numa_node = cpu_to_node(c->cpu);
1011 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1012 	if (err)
1013 		return err;
1014 	wq->db = &wq->db[MLX5_SND_DBR];
1015 
1016 	err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu));
1017 	if (err)
1018 		goto err_sq_wq_destroy;
1019 
1020 	return 0;
1021 
1022 err_sq_wq_destroy:
1023 	mlx5_wq_destroy(&sq->wq_ctrl);
1024 
1025 	return err;
1026 }
1027 
mlx5e_free_xdpsq(struct mlx5e_xdpsq * sq)1028 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq)
1029 {
1030 	mlx5e_free_xdpsq_db(sq);
1031 	mlx5_wq_destroy(&sq->wq_ctrl);
1032 }
1033 
mlx5e_free_icosq_db(struct mlx5e_icosq * sq)1034 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq)
1035 {
1036 	kvfree(sq->db.ico_wqe);
1037 }
1038 
mlx5e_alloc_icosq_db(struct mlx5e_icosq * sq,int numa)1039 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa)
1040 {
1041 	u8 wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1042 
1043 	sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz,
1044 						  sizeof(*sq->db.ico_wqe)),
1045 				       GFP_KERNEL, numa);
1046 	if (!sq->db.ico_wqe)
1047 		return -ENOMEM;
1048 
1049 	return 0;
1050 }
1051 
mlx5e_alloc_icosq(struct mlx5e_channel * c,struct mlx5e_sq_param * param,struct mlx5e_icosq * sq)1052 static int mlx5e_alloc_icosq(struct mlx5e_channel *c,
1053 			     struct mlx5e_sq_param *param,
1054 			     struct mlx5e_icosq *sq)
1055 {
1056 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1057 	struct mlx5_core_dev *mdev = c->mdev;
1058 	struct mlx5_wq_cyc *wq = &sq->wq;
1059 	int err;
1060 
1061 	sq->channel   = c;
1062 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1063 
1064 	param->wq.db_numa_node = cpu_to_node(c->cpu);
1065 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1066 	if (err)
1067 		return err;
1068 	wq->db = &wq->db[MLX5_SND_DBR];
1069 
1070 	err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu));
1071 	if (err)
1072 		goto err_sq_wq_destroy;
1073 
1074 	return 0;
1075 
1076 err_sq_wq_destroy:
1077 	mlx5_wq_destroy(&sq->wq_ctrl);
1078 
1079 	return err;
1080 }
1081 
mlx5e_free_icosq(struct mlx5e_icosq * sq)1082 static void mlx5e_free_icosq(struct mlx5e_icosq *sq)
1083 {
1084 	mlx5e_free_icosq_db(sq);
1085 	mlx5_wq_destroy(&sq->wq_ctrl);
1086 }
1087 
mlx5e_free_txqsq_db(struct mlx5e_txqsq * sq)1088 static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq)
1089 {
1090 	kvfree(sq->db.wqe_info);
1091 	kvfree(sq->db.dma_fifo);
1092 }
1093 
mlx5e_alloc_txqsq_db(struct mlx5e_txqsq * sq,int numa)1094 static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa)
1095 {
1096 	int wq_sz = mlx5_wq_cyc_get_size(&sq->wq);
1097 	int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS;
1098 
1099 	sq->db.dma_fifo = kvzalloc_node(array_size(df_sz,
1100 						   sizeof(*sq->db.dma_fifo)),
1101 					GFP_KERNEL, numa);
1102 	sq->db.wqe_info = kvzalloc_node(array_size(wq_sz,
1103 						   sizeof(*sq->db.wqe_info)),
1104 					GFP_KERNEL, numa);
1105 	if (!sq->db.dma_fifo || !sq->db.wqe_info) {
1106 		mlx5e_free_txqsq_db(sq);
1107 		return -ENOMEM;
1108 	}
1109 
1110 	sq->dma_fifo_mask = df_sz - 1;
1111 
1112 	return 0;
1113 }
1114 
1115 static void mlx5e_sq_recover(struct work_struct *work);
mlx5e_alloc_txqsq(struct mlx5e_channel * c,int txq_ix,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_txqsq * sq,int tc)1116 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c,
1117 			     int txq_ix,
1118 			     struct mlx5e_params *params,
1119 			     struct mlx5e_sq_param *param,
1120 			     struct mlx5e_txqsq *sq,
1121 			     int tc)
1122 {
1123 	void *sqc_wq               = MLX5_ADDR_OF(sqc, param->sqc, wq);
1124 	struct mlx5_core_dev *mdev = c->mdev;
1125 	struct mlx5_wq_cyc *wq = &sq->wq;
1126 	int err;
1127 
1128 	sq->pdev      = c->pdev;
1129 	sq->tstamp    = c->tstamp;
1130 	sq->clock     = &mdev->clock;
1131 	sq->mkey_be   = c->mkey_be;
1132 	sq->channel   = c;
1133 	sq->txq_ix    = txq_ix;
1134 	sq->uar_map   = mdev->mlx5e_res.bfreg.map;
1135 	sq->min_inline_mode = params->tx_min_inline_mode;
1136 	sq->stats     = &c->priv->channel_stats[c->ix].sq[tc];
1137 	INIT_WORK(&sq->recover.recover_work, mlx5e_sq_recover);
1138 	if (MLX5_IPSEC_DEV(c->priv->mdev))
1139 		set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state);
1140 	if (mlx5_accel_is_tls_device(c->priv->mdev))
1141 		set_bit(MLX5E_SQ_STATE_TLS, &sq->state);
1142 
1143 	param->wq.db_numa_node = cpu_to_node(c->cpu);
1144 	err = mlx5_wq_cyc_create(mdev, &param->wq, sqc_wq, wq, &sq->wq_ctrl);
1145 	if (err)
1146 		return err;
1147 	wq->db    = &wq->db[MLX5_SND_DBR];
1148 
1149 	err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu));
1150 	if (err)
1151 		goto err_sq_wq_destroy;
1152 
1153 	INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work);
1154 	sq->dim.mode = params->tx_cq_moderation.cq_period_mode;
1155 
1156 	return 0;
1157 
1158 err_sq_wq_destroy:
1159 	mlx5_wq_destroy(&sq->wq_ctrl);
1160 
1161 	return err;
1162 }
1163 
mlx5e_free_txqsq(struct mlx5e_txqsq * sq)1164 static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq)
1165 {
1166 	mlx5e_free_txqsq_db(sq);
1167 	mlx5_wq_destroy(&sq->wq_ctrl);
1168 }
1169 
1170 struct mlx5e_create_sq_param {
1171 	struct mlx5_wq_ctrl        *wq_ctrl;
1172 	u32                         cqn;
1173 	u32                         tisn;
1174 	u8                          tis_lst_sz;
1175 	u8                          min_inline_mode;
1176 };
1177 
mlx5e_create_sq(struct mlx5_core_dev * mdev,struct mlx5e_sq_param * param,struct mlx5e_create_sq_param * csp,u32 * sqn)1178 static int mlx5e_create_sq(struct mlx5_core_dev *mdev,
1179 			   struct mlx5e_sq_param *param,
1180 			   struct mlx5e_create_sq_param *csp,
1181 			   u32 *sqn)
1182 {
1183 	void *in;
1184 	void *sqc;
1185 	void *wq;
1186 	int inlen;
1187 	int err;
1188 
1189 	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1190 		sizeof(u64) * csp->wq_ctrl->buf.npages;
1191 	in = kvzalloc(inlen, GFP_KERNEL);
1192 	if (!in)
1193 		return -ENOMEM;
1194 
1195 	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1196 	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1197 
1198 	memcpy(sqc, param->sqc, sizeof(param->sqc));
1199 	MLX5_SET(sqc,  sqc, tis_lst_sz, csp->tis_lst_sz);
1200 	MLX5_SET(sqc,  sqc, tis_num_0, csp->tisn);
1201 	MLX5_SET(sqc,  sqc, cqn, csp->cqn);
1202 
1203 	if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT)
1204 		MLX5_SET(sqc,  sqc, min_wqe_inline_mode, csp->min_inline_mode);
1205 
1206 	MLX5_SET(sqc,  sqc, state, MLX5_SQC_STATE_RST);
1207 	MLX5_SET(sqc,  sqc, flush_in_error_en, 1);
1208 
1209 	MLX5_SET(wq,   wq, wq_type,       MLX5_WQ_TYPE_CYCLIC);
1210 	MLX5_SET(wq,   wq, uar_page,      mdev->mlx5e_res.bfreg.index);
1211 	MLX5_SET(wq,   wq, log_wq_pg_sz,  csp->wq_ctrl->buf.page_shift -
1212 					  MLX5_ADAPTER_PAGE_SHIFT);
1213 	MLX5_SET64(wq, wq, dbr_addr,      csp->wq_ctrl->db.dma);
1214 
1215 	mlx5_fill_page_frag_array(&csp->wq_ctrl->buf,
1216 				  (__be64 *)MLX5_ADDR_OF(wq, wq, pas));
1217 
1218 	err = mlx5_core_create_sq(mdev, in, inlen, sqn);
1219 
1220 	kvfree(in);
1221 
1222 	return err;
1223 }
1224 
1225 struct mlx5e_modify_sq_param {
1226 	int curr_state;
1227 	int next_state;
1228 	bool rl_update;
1229 	int rl_index;
1230 };
1231 
mlx5e_modify_sq(struct mlx5_core_dev * mdev,u32 sqn,struct mlx5e_modify_sq_param * p)1232 static int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn,
1233 			   struct mlx5e_modify_sq_param *p)
1234 {
1235 	void *in;
1236 	void *sqc;
1237 	int inlen;
1238 	int err;
1239 
1240 	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
1241 	in = kvzalloc(inlen, GFP_KERNEL);
1242 	if (!in)
1243 		return -ENOMEM;
1244 
1245 	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
1246 
1247 	MLX5_SET(modify_sq_in, in, sq_state, p->curr_state);
1248 	MLX5_SET(sqc, sqc, state, p->next_state);
1249 	if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) {
1250 		MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
1251 		MLX5_SET(sqc,  sqc, packet_pacing_rate_limit_index, p->rl_index);
1252 	}
1253 
1254 	err = mlx5_core_modify_sq(mdev, sqn, in, inlen);
1255 
1256 	kvfree(in);
1257 
1258 	return err;
1259 }
1260 
mlx5e_destroy_sq(struct mlx5_core_dev * mdev,u32 sqn)1261 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn)
1262 {
1263 	mlx5_core_destroy_sq(mdev, sqn);
1264 }
1265 
mlx5e_create_sq_rdy(struct mlx5_core_dev * mdev,struct mlx5e_sq_param * param,struct mlx5e_create_sq_param * csp,u32 * sqn)1266 static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev,
1267 			       struct mlx5e_sq_param *param,
1268 			       struct mlx5e_create_sq_param *csp,
1269 			       u32 *sqn)
1270 {
1271 	struct mlx5e_modify_sq_param msp = {0};
1272 	int err;
1273 
1274 	err = mlx5e_create_sq(mdev, param, csp, sqn);
1275 	if (err)
1276 		return err;
1277 
1278 	msp.curr_state = MLX5_SQC_STATE_RST;
1279 	msp.next_state = MLX5_SQC_STATE_RDY;
1280 	err = mlx5e_modify_sq(mdev, *sqn, &msp);
1281 	if (err)
1282 		mlx5e_destroy_sq(mdev, *sqn);
1283 
1284 	return err;
1285 }
1286 
1287 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1288 				struct mlx5e_txqsq *sq, u32 rate);
1289 
mlx5e_open_txqsq(struct mlx5e_channel * c,u32 tisn,int txq_ix,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_txqsq * sq,int tc)1290 static int mlx5e_open_txqsq(struct mlx5e_channel *c,
1291 			    u32 tisn,
1292 			    int txq_ix,
1293 			    struct mlx5e_params *params,
1294 			    struct mlx5e_sq_param *param,
1295 			    struct mlx5e_txqsq *sq,
1296 			    int tc)
1297 {
1298 	struct mlx5e_create_sq_param csp = {};
1299 	u32 tx_rate;
1300 	int err;
1301 
1302 	err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc);
1303 	if (err)
1304 		return err;
1305 
1306 	csp.tisn            = tisn;
1307 	csp.tis_lst_sz      = 1;
1308 	csp.cqn             = sq->cq.mcq.cqn;
1309 	csp.wq_ctrl         = &sq->wq_ctrl;
1310 	csp.min_inline_mode = sq->min_inline_mode;
1311 	err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
1312 	if (err)
1313 		goto err_free_txqsq;
1314 
1315 	tx_rate = c->priv->tx_rates[sq->txq_ix];
1316 	if (tx_rate)
1317 		mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate);
1318 
1319 	if (params->tx_dim_enabled)
1320 		sq->state |= BIT(MLX5E_SQ_STATE_AM);
1321 
1322 	return 0;
1323 
1324 err_free_txqsq:
1325 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1326 	mlx5e_free_txqsq(sq);
1327 
1328 	return err;
1329 }
1330 
mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq * sq)1331 static void mlx5e_reset_txqsq_cc_pc(struct mlx5e_txqsq *sq)
1332 {
1333 	WARN_ONCE(sq->cc != sq->pc,
1334 		  "SQ 0x%x: cc (0x%x) != pc (0x%x)\n",
1335 		  sq->sqn, sq->cc, sq->pc);
1336 	sq->cc = 0;
1337 	sq->dma_fifo_cc = 0;
1338 	sq->pc = 0;
1339 }
1340 
mlx5e_activate_txqsq(struct mlx5e_txqsq * sq)1341 static void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq)
1342 {
1343 	sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix);
1344 	clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state);
1345 	set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1346 	netdev_tx_reset_queue(sq->txq);
1347 	netif_tx_start_queue(sq->txq);
1348 }
1349 
netif_tx_disable_queue(struct netdev_queue * txq)1350 static inline void netif_tx_disable_queue(struct netdev_queue *txq)
1351 {
1352 	__netif_tx_lock_bh(txq);
1353 	netif_tx_stop_queue(txq);
1354 	__netif_tx_unlock_bh(txq);
1355 }
1356 
mlx5e_deactivate_txqsq(struct mlx5e_txqsq * sq)1357 static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq)
1358 {
1359 	struct mlx5e_channel *c = sq->channel;
1360 	struct mlx5_wq_cyc *wq = &sq->wq;
1361 
1362 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1363 	/* prevent netif_tx_wake_queue */
1364 	napi_synchronize(&c->napi);
1365 
1366 	netif_tx_disable_queue(sq->txq);
1367 
1368 	/* last doorbell out, godspeed .. */
1369 	if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) {
1370 		u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
1371 		struct mlx5e_tx_wqe *nop;
1372 
1373 		sq->db.wqe_info[pi].skb = NULL;
1374 		nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc);
1375 		mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl);
1376 	}
1377 }
1378 
mlx5e_close_txqsq(struct mlx5e_txqsq * sq)1379 static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq)
1380 {
1381 	struct mlx5e_channel *c = sq->channel;
1382 	struct mlx5_core_dev *mdev = c->mdev;
1383 	struct mlx5_rate_limit rl = {0};
1384 
1385 	mlx5e_destroy_sq(mdev, sq->sqn);
1386 	if (sq->rate_limit) {
1387 		rl.rate = sq->rate_limit;
1388 		mlx5_rl_remove_rate(mdev, &rl);
1389 	}
1390 	mlx5e_free_txqsq_descs(sq);
1391 	mlx5e_free_txqsq(sq);
1392 }
1393 
mlx5e_wait_for_sq_flush(struct mlx5e_txqsq * sq)1394 static int mlx5e_wait_for_sq_flush(struct mlx5e_txqsq *sq)
1395 {
1396 	unsigned long exp_time = jiffies + msecs_to_jiffies(2000);
1397 
1398 	while (time_before(jiffies, exp_time)) {
1399 		if (sq->cc == sq->pc)
1400 			return 0;
1401 
1402 		msleep(20);
1403 	}
1404 
1405 	netdev_err(sq->channel->netdev,
1406 		   "Wait for SQ 0x%x flush timeout (sq cc = 0x%x, sq pc = 0x%x)\n",
1407 		   sq->sqn, sq->cc, sq->pc);
1408 
1409 	return -ETIMEDOUT;
1410 }
1411 
mlx5e_sq_to_ready(struct mlx5e_txqsq * sq,int curr_state)1412 static int mlx5e_sq_to_ready(struct mlx5e_txqsq *sq, int curr_state)
1413 {
1414 	struct mlx5_core_dev *mdev = sq->channel->mdev;
1415 	struct net_device *dev = sq->channel->netdev;
1416 	struct mlx5e_modify_sq_param msp = {0};
1417 	int err;
1418 
1419 	msp.curr_state = curr_state;
1420 	msp.next_state = MLX5_SQC_STATE_RST;
1421 
1422 	err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1423 	if (err) {
1424 		netdev_err(dev, "Failed to move sq 0x%x to reset\n", sq->sqn);
1425 		return err;
1426 	}
1427 
1428 	memset(&msp, 0, sizeof(msp));
1429 	msp.curr_state = MLX5_SQC_STATE_RST;
1430 	msp.next_state = MLX5_SQC_STATE_RDY;
1431 
1432 	err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1433 	if (err) {
1434 		netdev_err(dev, "Failed to move sq 0x%x to ready\n", sq->sqn);
1435 		return err;
1436 	}
1437 
1438 	return 0;
1439 }
1440 
mlx5e_sq_recover(struct work_struct * work)1441 static void mlx5e_sq_recover(struct work_struct *work)
1442 {
1443 	struct mlx5e_txqsq_recover *recover =
1444 		container_of(work, struct mlx5e_txqsq_recover,
1445 			     recover_work);
1446 	struct mlx5e_txqsq *sq = container_of(recover, struct mlx5e_txqsq,
1447 					      recover);
1448 	struct mlx5_core_dev *mdev = sq->channel->mdev;
1449 	struct net_device *dev = sq->channel->netdev;
1450 	u8 state;
1451 	int err;
1452 
1453 	err = mlx5_core_query_sq_state(mdev, sq->sqn, &state);
1454 	if (err) {
1455 		netdev_err(dev, "Failed to query SQ 0x%x state. err = %d\n",
1456 			   sq->sqn, err);
1457 		return;
1458 	}
1459 
1460 	if (state != MLX5_RQC_STATE_ERR) {
1461 		netdev_err(dev, "SQ 0x%x not in ERROR state\n", sq->sqn);
1462 		return;
1463 	}
1464 
1465 	netif_tx_disable_queue(sq->txq);
1466 
1467 	if (mlx5e_wait_for_sq_flush(sq))
1468 		return;
1469 
1470 	/* If the interval between two consecutive recovers per SQ is too
1471 	 * short, don't recover to avoid infinite loop of ERR_CQE -> recover.
1472 	 * If we reached this state, there is probably a bug that needs to be
1473 	 * fixed. let's keep the queue close and let tx timeout cleanup.
1474 	 */
1475 	if (jiffies_to_msecs(jiffies - recover->last_recover) <
1476 	    MLX5E_SQ_RECOVER_MIN_INTERVAL) {
1477 		netdev_err(dev, "Recover SQ 0x%x canceled, too many error CQEs\n",
1478 			   sq->sqn);
1479 		return;
1480 	}
1481 
1482 	/* At this point, no new packets will arrive from the stack as TXQ is
1483 	 * marked with QUEUE_STATE_DRV_XOFF. In addition, NAPI cleared all
1484 	 * pending WQEs.  SQ can safely reset the SQ.
1485 	 */
1486 	if (mlx5e_sq_to_ready(sq, state))
1487 		return;
1488 
1489 	mlx5e_reset_txqsq_cc_pc(sq);
1490 	sq->stats->recover++;
1491 	recover->last_recover = jiffies;
1492 	mlx5e_activate_txqsq(sq);
1493 }
1494 
mlx5e_open_icosq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_icosq * sq)1495 static int mlx5e_open_icosq(struct mlx5e_channel *c,
1496 			    struct mlx5e_params *params,
1497 			    struct mlx5e_sq_param *param,
1498 			    struct mlx5e_icosq *sq)
1499 {
1500 	struct mlx5e_create_sq_param csp = {};
1501 	int err;
1502 
1503 	err = mlx5e_alloc_icosq(c, param, sq);
1504 	if (err)
1505 		return err;
1506 
1507 	csp.cqn             = sq->cq.mcq.cqn;
1508 	csp.wq_ctrl         = &sq->wq_ctrl;
1509 	csp.min_inline_mode = params->tx_min_inline_mode;
1510 	set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1511 	err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
1512 	if (err)
1513 		goto err_free_icosq;
1514 
1515 	return 0;
1516 
1517 err_free_icosq:
1518 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1519 	mlx5e_free_icosq(sq);
1520 
1521 	return err;
1522 }
1523 
mlx5e_close_icosq(struct mlx5e_icosq * sq)1524 static void mlx5e_close_icosq(struct mlx5e_icosq *sq)
1525 {
1526 	struct mlx5e_channel *c = sq->channel;
1527 
1528 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1529 	napi_synchronize(&c->napi);
1530 
1531 	mlx5e_destroy_sq(c->mdev, sq->sqn);
1532 	mlx5e_free_icosq(sq);
1533 }
1534 
mlx5e_open_xdpsq(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_sq_param * param,struct mlx5e_xdpsq * sq,bool is_redirect)1535 static int mlx5e_open_xdpsq(struct mlx5e_channel *c,
1536 			    struct mlx5e_params *params,
1537 			    struct mlx5e_sq_param *param,
1538 			    struct mlx5e_xdpsq *sq,
1539 			    bool is_redirect)
1540 {
1541 	unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT;
1542 	struct mlx5e_create_sq_param csp = {};
1543 	unsigned int inline_hdr_sz = 0;
1544 	int err;
1545 	int i;
1546 
1547 	err = mlx5e_alloc_xdpsq(c, params, param, sq, is_redirect);
1548 	if (err)
1549 		return err;
1550 
1551 	csp.tis_lst_sz      = 1;
1552 	csp.tisn            = c->priv->tisn[0]; /* tc = 0 */
1553 	csp.cqn             = sq->cq.mcq.cqn;
1554 	csp.wq_ctrl         = &sq->wq_ctrl;
1555 	csp.min_inline_mode = sq->min_inline_mode;
1556 	if (is_redirect)
1557 		set_bit(MLX5E_SQ_STATE_REDIRECT, &sq->state);
1558 	set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1559 	err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn);
1560 	if (err)
1561 		goto err_free_xdpsq;
1562 
1563 	if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) {
1564 		inline_hdr_sz = MLX5E_XDP_MIN_INLINE;
1565 		ds_cnt++;
1566 	}
1567 
1568 	/* Pre initialize fixed WQE fields */
1569 	for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) {
1570 		struct mlx5e_tx_wqe      *wqe  = mlx5_wq_cyc_get_wqe(&sq->wq, i);
1571 		struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl;
1572 		struct mlx5_wqe_eth_seg  *eseg = &wqe->eth;
1573 		struct mlx5_wqe_data_seg *dseg;
1574 
1575 		cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt);
1576 		eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz);
1577 
1578 		dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1);
1579 		dseg->lkey = sq->mkey_be;
1580 	}
1581 
1582 	return 0;
1583 
1584 err_free_xdpsq:
1585 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1586 	mlx5e_free_xdpsq(sq);
1587 
1588 	return err;
1589 }
1590 
mlx5e_close_xdpsq(struct mlx5e_xdpsq * sq)1591 static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq)
1592 {
1593 	struct mlx5e_channel *c = sq->channel;
1594 
1595 	clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
1596 	napi_synchronize(&c->napi);
1597 
1598 	mlx5e_destroy_sq(c->mdev, sq->sqn);
1599 	mlx5e_free_xdpsq_descs(sq);
1600 	mlx5e_free_xdpsq(sq);
1601 }
1602 
mlx5e_alloc_cq_common(struct mlx5_core_dev * mdev,struct mlx5e_cq_param * param,struct mlx5e_cq * cq)1603 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev,
1604 				 struct mlx5e_cq_param *param,
1605 				 struct mlx5e_cq *cq)
1606 {
1607 	struct mlx5_core_cq *mcq = &cq->mcq;
1608 	int eqn_not_used;
1609 	unsigned int irqn;
1610 	int err;
1611 	u32 i;
1612 
1613 	err = mlx5_cqwq_create(mdev, &param->wq, param->cqc, &cq->wq,
1614 			       &cq->wq_ctrl);
1615 	if (err)
1616 		return err;
1617 
1618 	mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn);
1619 
1620 	mcq->cqe_sz     = 64;
1621 	mcq->set_ci_db  = cq->wq_ctrl.db.db;
1622 	mcq->arm_db     = cq->wq_ctrl.db.db + 1;
1623 	*mcq->set_ci_db = 0;
1624 	*mcq->arm_db    = 0;
1625 	mcq->vector     = param->eq_ix;
1626 	mcq->comp       = mlx5e_completion_event;
1627 	mcq->event      = mlx5e_cq_error_event;
1628 	mcq->irqn       = irqn;
1629 
1630 	for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) {
1631 		struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i);
1632 
1633 		cqe->op_own = 0xf1;
1634 	}
1635 
1636 	cq->mdev = mdev;
1637 
1638 	return 0;
1639 }
1640 
mlx5e_alloc_cq(struct mlx5e_channel * c,struct mlx5e_cq_param * param,struct mlx5e_cq * cq)1641 static int mlx5e_alloc_cq(struct mlx5e_channel *c,
1642 			  struct mlx5e_cq_param *param,
1643 			  struct mlx5e_cq *cq)
1644 {
1645 	struct mlx5_core_dev *mdev = c->priv->mdev;
1646 	int err;
1647 
1648 	param->wq.buf_numa_node = cpu_to_node(c->cpu);
1649 	param->wq.db_numa_node  = cpu_to_node(c->cpu);
1650 	param->eq_ix   = c->ix;
1651 
1652 	err = mlx5e_alloc_cq_common(mdev, param, cq);
1653 
1654 	cq->napi    = &c->napi;
1655 	cq->channel = c;
1656 
1657 	return err;
1658 }
1659 
mlx5e_free_cq(struct mlx5e_cq * cq)1660 static void mlx5e_free_cq(struct mlx5e_cq *cq)
1661 {
1662 	mlx5_wq_destroy(&cq->wq_ctrl);
1663 }
1664 
mlx5e_create_cq(struct mlx5e_cq * cq,struct mlx5e_cq_param * param)1665 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param)
1666 {
1667 	struct mlx5_core_dev *mdev = cq->mdev;
1668 	struct mlx5_core_cq *mcq = &cq->mcq;
1669 
1670 	void *in;
1671 	void *cqc;
1672 	int inlen;
1673 	unsigned int irqn_not_used;
1674 	int eqn;
1675 	int err;
1676 
1677 	inlen = MLX5_ST_SZ_BYTES(create_cq_in) +
1678 		sizeof(u64) * cq->wq_ctrl.buf.npages;
1679 	in = kvzalloc(inlen, GFP_KERNEL);
1680 	if (!in)
1681 		return -ENOMEM;
1682 
1683 	cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
1684 
1685 	memcpy(cqc, param->cqc, sizeof(param->cqc));
1686 
1687 	mlx5_fill_page_frag_array(&cq->wq_ctrl.buf,
1688 				  (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas));
1689 
1690 	mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used);
1691 
1692 	MLX5_SET(cqc,   cqc, cq_period_mode, param->cq_period_mode);
1693 	MLX5_SET(cqc,   cqc, c_eqn,         eqn);
1694 	MLX5_SET(cqc,   cqc, uar_page,      mdev->priv.uar->index);
1695 	MLX5_SET(cqc,   cqc, log_page_size, cq->wq_ctrl.buf.page_shift -
1696 					    MLX5_ADAPTER_PAGE_SHIFT);
1697 	MLX5_SET64(cqc, cqc, dbr_addr,      cq->wq_ctrl.db.dma);
1698 
1699 	err = mlx5_core_create_cq(mdev, mcq, in, inlen);
1700 
1701 	kvfree(in);
1702 
1703 	if (err)
1704 		return err;
1705 
1706 	mlx5e_cq_arm(cq);
1707 
1708 	return 0;
1709 }
1710 
mlx5e_destroy_cq(struct mlx5e_cq * cq)1711 static void mlx5e_destroy_cq(struct mlx5e_cq *cq)
1712 {
1713 	mlx5_core_destroy_cq(cq->mdev, &cq->mcq);
1714 }
1715 
mlx5e_open_cq(struct mlx5e_channel * c,struct net_dim_cq_moder moder,struct mlx5e_cq_param * param,struct mlx5e_cq * cq)1716 static int mlx5e_open_cq(struct mlx5e_channel *c,
1717 			 struct net_dim_cq_moder moder,
1718 			 struct mlx5e_cq_param *param,
1719 			 struct mlx5e_cq *cq)
1720 {
1721 	struct mlx5_core_dev *mdev = c->mdev;
1722 	int err;
1723 
1724 	err = mlx5e_alloc_cq(c, param, cq);
1725 	if (err)
1726 		return err;
1727 
1728 	err = mlx5e_create_cq(cq, param);
1729 	if (err)
1730 		goto err_free_cq;
1731 
1732 	if (MLX5_CAP_GEN(mdev, cq_moderation))
1733 		mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts);
1734 	return 0;
1735 
1736 err_free_cq:
1737 	mlx5e_free_cq(cq);
1738 
1739 	return err;
1740 }
1741 
mlx5e_close_cq(struct mlx5e_cq * cq)1742 static void mlx5e_close_cq(struct mlx5e_cq *cq)
1743 {
1744 	mlx5e_destroy_cq(cq);
1745 	mlx5e_free_cq(cq);
1746 }
1747 
mlx5e_get_cpu(struct mlx5e_priv * priv,int ix)1748 static int mlx5e_get_cpu(struct mlx5e_priv *priv, int ix)
1749 {
1750 	return cpumask_first(priv->mdev->priv.irq_info[ix].mask);
1751 }
1752 
mlx5e_open_tx_cqs(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_channel_param * cparam)1753 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c,
1754 			     struct mlx5e_params *params,
1755 			     struct mlx5e_channel_param *cparam)
1756 {
1757 	int err;
1758 	int tc;
1759 
1760 	for (tc = 0; tc < c->num_tc; tc++) {
1761 		err = mlx5e_open_cq(c, params->tx_cq_moderation,
1762 				    &cparam->tx_cq, &c->sq[tc].cq);
1763 		if (err)
1764 			goto err_close_tx_cqs;
1765 	}
1766 
1767 	return 0;
1768 
1769 err_close_tx_cqs:
1770 	for (tc--; tc >= 0; tc--)
1771 		mlx5e_close_cq(&c->sq[tc].cq);
1772 
1773 	return err;
1774 }
1775 
mlx5e_close_tx_cqs(struct mlx5e_channel * c)1776 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c)
1777 {
1778 	int tc;
1779 
1780 	for (tc = 0; tc < c->num_tc; tc++)
1781 		mlx5e_close_cq(&c->sq[tc].cq);
1782 }
1783 
mlx5e_open_sqs(struct mlx5e_channel * c,struct mlx5e_params * params,struct mlx5e_channel_param * cparam)1784 static int mlx5e_open_sqs(struct mlx5e_channel *c,
1785 			  struct mlx5e_params *params,
1786 			  struct mlx5e_channel_param *cparam)
1787 {
1788 	struct mlx5e_priv *priv = c->priv;
1789 	int err, tc, max_nch = priv->profile->max_nch(priv->mdev);
1790 
1791 	for (tc = 0; tc < params->num_tc; tc++) {
1792 		int txq_ix = c->ix + tc * max_nch;
1793 
1794 		err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix,
1795 				       params, &cparam->sq, &c->sq[tc], tc);
1796 		if (err)
1797 			goto err_close_sqs;
1798 	}
1799 
1800 	return 0;
1801 
1802 err_close_sqs:
1803 	for (tc--; tc >= 0; tc--)
1804 		mlx5e_close_txqsq(&c->sq[tc]);
1805 
1806 	return err;
1807 }
1808 
mlx5e_close_sqs(struct mlx5e_channel * c)1809 static void mlx5e_close_sqs(struct mlx5e_channel *c)
1810 {
1811 	int tc;
1812 
1813 	for (tc = 0; tc < c->num_tc; tc++)
1814 		mlx5e_close_txqsq(&c->sq[tc]);
1815 }
1816 
mlx5e_set_sq_maxrate(struct net_device * dev,struct mlx5e_txqsq * sq,u32 rate)1817 static int mlx5e_set_sq_maxrate(struct net_device *dev,
1818 				struct mlx5e_txqsq *sq, u32 rate)
1819 {
1820 	struct mlx5e_priv *priv = netdev_priv(dev);
1821 	struct mlx5_core_dev *mdev = priv->mdev;
1822 	struct mlx5e_modify_sq_param msp = {0};
1823 	struct mlx5_rate_limit rl = {0};
1824 	u16 rl_index = 0;
1825 	int err;
1826 
1827 	if (rate == sq->rate_limit)
1828 		/* nothing to do */
1829 		return 0;
1830 
1831 	if (sq->rate_limit) {
1832 		rl.rate = sq->rate_limit;
1833 		/* remove current rl index to free space to next ones */
1834 		mlx5_rl_remove_rate(mdev, &rl);
1835 	}
1836 
1837 	sq->rate_limit = 0;
1838 
1839 	if (rate) {
1840 		rl.rate = rate;
1841 		err = mlx5_rl_add_rate(mdev, &rl_index, &rl);
1842 		if (err) {
1843 			netdev_err(dev, "Failed configuring rate %u: %d\n",
1844 				   rate, err);
1845 			return err;
1846 		}
1847 	}
1848 
1849 	msp.curr_state = MLX5_SQC_STATE_RDY;
1850 	msp.next_state = MLX5_SQC_STATE_RDY;
1851 	msp.rl_index   = rl_index;
1852 	msp.rl_update  = true;
1853 	err = mlx5e_modify_sq(mdev, sq->sqn, &msp);
1854 	if (err) {
1855 		netdev_err(dev, "Failed configuring rate %u: %d\n",
1856 			   rate, err);
1857 		/* remove the rate from the table */
1858 		if (rate)
1859 			mlx5_rl_remove_rate(mdev, &rl);
1860 		return err;
1861 	}
1862 
1863 	sq->rate_limit = rate;
1864 	return 0;
1865 }
1866 
mlx5e_set_tx_maxrate(struct net_device * dev,int index,u32 rate)1867 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
1868 {
1869 	struct mlx5e_priv *priv = netdev_priv(dev);
1870 	struct mlx5_core_dev *mdev = priv->mdev;
1871 	struct mlx5e_txqsq *sq = priv->txq2sq[index];
1872 	int err = 0;
1873 
1874 	if (!mlx5_rl_is_supported(mdev)) {
1875 		netdev_err(dev, "Rate limiting is not supported on this device\n");
1876 		return -EINVAL;
1877 	}
1878 
1879 	/* rate is given in Mb/sec, HW config is in Kb/sec */
1880 	rate = rate << 10;
1881 
1882 	/* Check whether rate in valid range, 0 is always valid */
1883 	if (rate && !mlx5_rl_is_in_range(mdev, rate)) {
1884 		netdev_err(dev, "TX rate %u, is not in range\n", rate);
1885 		return -ERANGE;
1886 	}
1887 
1888 	mutex_lock(&priv->state_lock);
1889 	if (test_bit(MLX5E_STATE_OPENED, &priv->state))
1890 		err = mlx5e_set_sq_maxrate(dev, sq, rate);
1891 	if (!err)
1892 		priv->tx_rates[index] = rate;
1893 	mutex_unlock(&priv->state_lock);
1894 
1895 	return err;
1896 }
1897 
mlx5e_open_channel(struct mlx5e_priv * priv,int ix,struct mlx5e_params * params,struct mlx5e_channel_param * cparam,struct mlx5e_channel ** cp)1898 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
1899 			      struct mlx5e_params *params,
1900 			      struct mlx5e_channel_param *cparam,
1901 			      struct mlx5e_channel **cp)
1902 {
1903 	struct net_dim_cq_moder icocq_moder = {0, 0};
1904 	struct net_device *netdev = priv->netdev;
1905 	int cpu = mlx5e_get_cpu(priv, ix);
1906 	struct mlx5e_channel *c;
1907 	unsigned int irq;
1908 	int err;
1909 	int eqn;
1910 
1911 	c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu));
1912 	if (!c)
1913 		return -ENOMEM;
1914 
1915 	c->priv     = priv;
1916 	c->mdev     = priv->mdev;
1917 	c->tstamp   = &priv->tstamp;
1918 	c->ix       = ix;
1919 	c->cpu      = cpu;
1920 	c->pdev     = &priv->mdev->pdev->dev;
1921 	c->netdev   = priv->netdev;
1922 	c->mkey_be  = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key);
1923 	c->num_tc   = params->num_tc;
1924 	c->xdp      = !!params->xdp_prog;
1925 	c->stats    = &priv->channel_stats[ix].ch;
1926 
1927 	mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq);
1928 	c->irq_desc = irq_to_desc(irq);
1929 
1930 	netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
1931 
1932 	err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
1933 	if (err)
1934 		goto err_napi_del;
1935 
1936 	err = mlx5e_open_tx_cqs(c, params, cparam);
1937 	if (err)
1938 		goto err_close_icosq_cq;
1939 
1940 	err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq);
1941 	if (err)
1942 		goto err_close_tx_cqs;
1943 
1944 	err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq);
1945 	if (err)
1946 		goto err_close_xdp_tx_cqs;
1947 
1948 	/* XDP SQ CQ params are same as normal TXQ sq CQ params */
1949 	err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation,
1950 				     &cparam->tx_cq, &c->rq.xdpsq.cq) : 0;
1951 	if (err)
1952 		goto err_close_rx_cq;
1953 
1954 	napi_enable(&c->napi);
1955 
1956 	err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq);
1957 	if (err)
1958 		goto err_disable_napi;
1959 
1960 	err = mlx5e_open_sqs(c, params, cparam);
1961 	if (err)
1962 		goto err_close_icosq;
1963 
1964 	err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0;
1965 	if (err)
1966 		goto err_close_sqs;
1967 
1968 	err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq);
1969 	if (err)
1970 		goto err_close_xdp_sq;
1971 
1972 	err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->xdpsq, true);
1973 	if (err)
1974 		goto err_close_rq;
1975 
1976 	*cp = c;
1977 
1978 	return 0;
1979 
1980 err_close_rq:
1981 	mlx5e_close_rq(&c->rq);
1982 
1983 err_close_xdp_sq:
1984 	if (c->xdp)
1985 		mlx5e_close_xdpsq(&c->rq.xdpsq);
1986 
1987 err_close_sqs:
1988 	mlx5e_close_sqs(c);
1989 
1990 err_close_icosq:
1991 	mlx5e_close_icosq(&c->icosq);
1992 
1993 err_disable_napi:
1994 	napi_disable(&c->napi);
1995 	if (c->xdp)
1996 		mlx5e_close_cq(&c->rq.xdpsq.cq);
1997 
1998 err_close_rx_cq:
1999 	mlx5e_close_cq(&c->rq.cq);
2000 
2001 err_close_xdp_tx_cqs:
2002 	mlx5e_close_cq(&c->xdpsq.cq);
2003 
2004 err_close_tx_cqs:
2005 	mlx5e_close_tx_cqs(c);
2006 
2007 err_close_icosq_cq:
2008 	mlx5e_close_cq(&c->icosq.cq);
2009 
2010 err_napi_del:
2011 	netif_napi_del(&c->napi);
2012 	kvfree(c);
2013 
2014 	return err;
2015 }
2016 
mlx5e_activate_channel(struct mlx5e_channel * c)2017 static void mlx5e_activate_channel(struct mlx5e_channel *c)
2018 {
2019 	int tc;
2020 
2021 	for (tc = 0; tc < c->num_tc; tc++)
2022 		mlx5e_activate_txqsq(&c->sq[tc]);
2023 	mlx5e_activate_rq(&c->rq);
2024 	netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
2025 }
2026 
mlx5e_deactivate_channel(struct mlx5e_channel * c)2027 static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
2028 {
2029 	int tc;
2030 
2031 	mlx5e_deactivate_rq(&c->rq);
2032 	for (tc = 0; tc < c->num_tc; tc++)
2033 		mlx5e_deactivate_txqsq(&c->sq[tc]);
2034 }
2035 
mlx5e_close_channel(struct mlx5e_channel * c)2036 static void mlx5e_close_channel(struct mlx5e_channel *c)
2037 {
2038 	mlx5e_close_xdpsq(&c->xdpsq);
2039 	mlx5e_close_rq(&c->rq);
2040 	if (c->xdp)
2041 		mlx5e_close_xdpsq(&c->rq.xdpsq);
2042 	mlx5e_close_sqs(c);
2043 	mlx5e_close_icosq(&c->icosq);
2044 	napi_disable(&c->napi);
2045 	if (c->xdp)
2046 		mlx5e_close_cq(&c->rq.xdpsq.cq);
2047 	mlx5e_close_cq(&c->rq.cq);
2048 	mlx5e_close_cq(&c->xdpsq.cq);
2049 	mlx5e_close_tx_cqs(c);
2050 	mlx5e_close_cq(&c->icosq.cq);
2051 	netif_napi_del(&c->napi);
2052 
2053 	kvfree(c);
2054 }
2055 
2056 #define DEFAULT_FRAG_SIZE (2048)
2057 
mlx5e_build_rq_frags_info(struct mlx5_core_dev * mdev,struct mlx5e_params * params,struct mlx5e_rq_frags_info * info)2058 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev,
2059 				      struct mlx5e_params *params,
2060 				      struct mlx5e_rq_frags_info *info)
2061 {
2062 	u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu);
2063 	int frag_size_max = DEFAULT_FRAG_SIZE;
2064 	u32 buf_size = 0;
2065 	int i;
2066 
2067 #ifdef CONFIG_MLX5_EN_IPSEC
2068 	if (MLX5_IPSEC_DEV(mdev))
2069 		byte_count += MLX5E_METADATA_ETHER_LEN;
2070 #endif
2071 
2072 	if (mlx5e_rx_is_linear_skb(mdev, params)) {
2073 		int frag_stride;
2074 
2075 		frag_stride = mlx5e_rx_get_linear_frag_sz(params);
2076 		frag_stride = roundup_pow_of_two(frag_stride);
2077 
2078 		info->arr[0].frag_size = byte_count;
2079 		info->arr[0].frag_stride = frag_stride;
2080 		info->num_frags = 1;
2081 		info->wqe_bulk = PAGE_SIZE / frag_stride;
2082 		goto out;
2083 	}
2084 
2085 	if (byte_count > PAGE_SIZE +
2086 	    (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max)
2087 		frag_size_max = PAGE_SIZE;
2088 
2089 	i = 0;
2090 	while (buf_size < byte_count) {
2091 		int frag_size = byte_count - buf_size;
2092 
2093 		if (i < MLX5E_MAX_RX_FRAGS - 1)
2094 			frag_size = min(frag_size, frag_size_max);
2095 
2096 		info->arr[i].frag_size = frag_size;
2097 		info->arr[i].frag_stride = roundup_pow_of_two(frag_size);
2098 
2099 		buf_size += frag_size;
2100 		i++;
2101 	}
2102 	info->num_frags = i;
2103 	/* number of different wqes sharing a page */
2104 	info->wqe_bulk = 1 + (info->num_frags % 2);
2105 
2106 out:
2107 	info->wqe_bulk = max_t(u8, info->wqe_bulk, 8);
2108 	info->log_num_frags = order_base_2(info->num_frags);
2109 }
2110 
mlx5e_get_rqwq_log_stride(u8 wq_type,int ndsegs)2111 static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs)
2112 {
2113 	int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs;
2114 
2115 	switch (wq_type) {
2116 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2117 		sz += sizeof(struct mlx5e_rx_wqe_ll);
2118 		break;
2119 	default: /* MLX5_WQ_TYPE_CYCLIC */
2120 		sz += sizeof(struct mlx5e_rx_wqe_cyc);
2121 	}
2122 
2123 	return order_base_2(sz);
2124 }
2125 
mlx5e_build_rq_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_rq_param * param)2126 static void mlx5e_build_rq_param(struct mlx5e_priv *priv,
2127 				 struct mlx5e_params *params,
2128 				 struct mlx5e_rq_param *param)
2129 {
2130 	struct mlx5_core_dev *mdev = priv->mdev;
2131 	void *rqc = param->rqc;
2132 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2133 	int ndsegs = 1;
2134 
2135 	switch (params->rq_wq_type) {
2136 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2137 		MLX5_SET(wq, wq, log_wqe_num_of_strides,
2138 			 mlx5e_mpwqe_get_log_num_strides(mdev, params) -
2139 			 MLX5_MPWQE_LOG_NUM_STRIDES_BASE);
2140 		MLX5_SET(wq, wq, log_wqe_stride_size,
2141 			 mlx5e_mpwqe_get_log_stride_size(mdev, params) -
2142 			 MLX5_MPWQE_LOG_STRIDE_SZ_BASE);
2143 		MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params));
2144 		break;
2145 	default: /* MLX5_WQ_TYPE_CYCLIC */
2146 		MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames);
2147 		mlx5e_build_rq_frags_info(mdev, params, &param->frags_info);
2148 		ndsegs = param->frags_info.num_frags;
2149 	}
2150 
2151 	MLX5_SET(wq, wq, wq_type,          params->rq_wq_type);
2152 	MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
2153 	MLX5_SET(wq, wq, log_wq_stride,
2154 		 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs));
2155 	MLX5_SET(wq, wq, pd,               mdev->mlx5e_res.pdn);
2156 	MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter);
2157 	MLX5_SET(rqc, rqc, vsd,            params->vlan_strip_disable);
2158 	MLX5_SET(rqc, rqc, scatter_fcs,    params->scatter_fcs_en);
2159 
2160 	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
2161 }
2162 
mlx5e_build_drop_rq_param(struct mlx5e_priv * priv,struct mlx5e_rq_param * param)2163 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv,
2164 				      struct mlx5e_rq_param *param)
2165 {
2166 	struct mlx5_core_dev *mdev = priv->mdev;
2167 	void *rqc = param->rqc;
2168 	void *wq = MLX5_ADDR_OF(rqc, rqc, wq);
2169 
2170 	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
2171 	MLX5_SET(wq, wq, log_wq_stride,
2172 		 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1));
2173 	MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter);
2174 
2175 	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
2176 }
2177 
mlx5e_build_sq_param_common(struct mlx5e_priv * priv,struct mlx5e_sq_param * param)2178 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv,
2179 					struct mlx5e_sq_param *param)
2180 {
2181 	void *sqc = param->sqc;
2182 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2183 
2184 	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
2185 	MLX5_SET(wq, wq, pd,            priv->mdev->mlx5e_res.pdn);
2186 
2187 	param->wq.buf_numa_node = dev_to_node(&priv->mdev->pdev->dev);
2188 }
2189 
mlx5e_build_sq_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_sq_param * param)2190 static void mlx5e_build_sq_param(struct mlx5e_priv *priv,
2191 				 struct mlx5e_params *params,
2192 				 struct mlx5e_sq_param *param)
2193 {
2194 	void *sqc = param->sqc;
2195 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2196 
2197 	mlx5e_build_sq_param_common(priv, param);
2198 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2199 	MLX5_SET(sqc, sqc, allow_swp, !!MLX5_IPSEC_DEV(priv->mdev));
2200 }
2201 
mlx5e_build_common_cq_param(struct mlx5e_priv * priv,struct mlx5e_cq_param * param)2202 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv,
2203 					struct mlx5e_cq_param *param)
2204 {
2205 	void *cqc = param->cqc;
2206 
2207 	MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index);
2208 }
2209 
mlx5e_build_rx_cq_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_cq_param * param)2210 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv,
2211 				    struct mlx5e_params *params,
2212 				    struct mlx5e_cq_param *param)
2213 {
2214 	struct mlx5_core_dev *mdev = priv->mdev;
2215 	void *cqc = param->cqc;
2216 	u8 log_cq_size;
2217 
2218 	switch (params->rq_wq_type) {
2219 	case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ:
2220 		log_cq_size = mlx5e_mpwqe_get_log_rq_size(params) +
2221 			mlx5e_mpwqe_get_log_num_strides(mdev, params);
2222 		break;
2223 	default: /* MLX5_WQ_TYPE_CYCLIC */
2224 		log_cq_size = params->log_rq_mtu_frames;
2225 	}
2226 
2227 	MLX5_SET(cqc, cqc, log_cq_size, log_cq_size);
2228 	if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) {
2229 		MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM);
2230 		MLX5_SET(cqc, cqc, cqe_comp_en, 1);
2231 	}
2232 
2233 	mlx5e_build_common_cq_param(priv, param);
2234 	param->cq_period_mode = params->rx_cq_moderation.cq_period_mode;
2235 }
2236 
mlx5e_build_tx_cq_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_cq_param * param)2237 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv,
2238 				    struct mlx5e_params *params,
2239 				    struct mlx5e_cq_param *param)
2240 {
2241 	void *cqc = param->cqc;
2242 
2243 	MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size);
2244 
2245 	mlx5e_build_common_cq_param(priv, param);
2246 	param->cq_period_mode = params->tx_cq_moderation.cq_period_mode;
2247 }
2248 
mlx5e_build_ico_cq_param(struct mlx5e_priv * priv,u8 log_wq_size,struct mlx5e_cq_param * param)2249 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv,
2250 				     u8 log_wq_size,
2251 				     struct mlx5e_cq_param *param)
2252 {
2253 	void *cqc = param->cqc;
2254 
2255 	MLX5_SET(cqc, cqc, log_cq_size, log_wq_size);
2256 
2257 	mlx5e_build_common_cq_param(priv, param);
2258 
2259 	param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
2260 }
2261 
mlx5e_build_icosq_param(struct mlx5e_priv * priv,u8 log_wq_size,struct mlx5e_sq_param * param)2262 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv,
2263 				    u8 log_wq_size,
2264 				    struct mlx5e_sq_param *param)
2265 {
2266 	void *sqc = param->sqc;
2267 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2268 
2269 	mlx5e_build_sq_param_common(priv, param);
2270 
2271 	MLX5_SET(wq, wq, log_wq_sz, log_wq_size);
2272 	MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq));
2273 }
2274 
mlx5e_build_xdpsq_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_sq_param * param)2275 static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv,
2276 				    struct mlx5e_params *params,
2277 				    struct mlx5e_sq_param *param)
2278 {
2279 	void *sqc = param->sqc;
2280 	void *wq = MLX5_ADDR_OF(sqc, sqc, wq);
2281 
2282 	mlx5e_build_sq_param_common(priv, param);
2283 	MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size);
2284 }
2285 
mlx5e_build_channel_param(struct mlx5e_priv * priv,struct mlx5e_params * params,struct mlx5e_channel_param * cparam)2286 static void mlx5e_build_channel_param(struct mlx5e_priv *priv,
2287 				      struct mlx5e_params *params,
2288 				      struct mlx5e_channel_param *cparam)
2289 {
2290 	u8 icosq_log_wq_sz = MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE;
2291 
2292 	mlx5e_build_rq_param(priv, params, &cparam->rq);
2293 	mlx5e_build_sq_param(priv, params, &cparam->sq);
2294 	mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq);
2295 	mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq);
2296 	mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq);
2297 	mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq);
2298 	mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq);
2299 }
2300 
mlx5e_open_channels(struct mlx5e_priv * priv,struct mlx5e_channels * chs)2301 int mlx5e_open_channels(struct mlx5e_priv *priv,
2302 			struct mlx5e_channels *chs)
2303 {
2304 	struct mlx5e_channel_param *cparam;
2305 	int err = -ENOMEM;
2306 	int i;
2307 
2308 	chs->num = chs->params.num_channels;
2309 
2310 	chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL);
2311 	cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL);
2312 	if (!chs->c || !cparam)
2313 		goto err_free;
2314 
2315 	mlx5e_build_channel_param(priv, &chs->params, cparam);
2316 	for (i = 0; i < chs->num; i++) {
2317 		err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]);
2318 		if (err)
2319 			goto err_close_channels;
2320 	}
2321 
2322 	kvfree(cparam);
2323 	return 0;
2324 
2325 err_close_channels:
2326 	for (i--; i >= 0; i--)
2327 		mlx5e_close_channel(chs->c[i]);
2328 
2329 err_free:
2330 	kfree(chs->c);
2331 	kvfree(cparam);
2332 	chs->num = 0;
2333 	return err;
2334 }
2335 
mlx5e_activate_channels(struct mlx5e_channels * chs)2336 static void mlx5e_activate_channels(struct mlx5e_channels *chs)
2337 {
2338 	int i;
2339 
2340 	for (i = 0; i < chs->num; i++)
2341 		mlx5e_activate_channel(chs->c[i]);
2342 }
2343 
mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels * chs)2344 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs)
2345 {
2346 	int err = 0;
2347 	int i;
2348 
2349 	for (i = 0; i < chs->num; i++)
2350 		err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq,
2351 						  err ? 0 : 20000);
2352 
2353 	return err ? -ETIMEDOUT : 0;
2354 }
2355 
mlx5e_deactivate_channels(struct mlx5e_channels * chs)2356 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs)
2357 {
2358 	int i;
2359 
2360 	for (i = 0; i < chs->num; i++)
2361 		mlx5e_deactivate_channel(chs->c[i]);
2362 }
2363 
mlx5e_close_channels(struct mlx5e_channels * chs)2364 void mlx5e_close_channels(struct mlx5e_channels *chs)
2365 {
2366 	int i;
2367 
2368 	for (i = 0; i < chs->num; i++)
2369 		mlx5e_close_channel(chs->c[i]);
2370 
2371 	kfree(chs->c);
2372 	chs->num = 0;
2373 }
2374 
2375 static int
mlx5e_create_rqt(struct mlx5e_priv * priv,int sz,struct mlx5e_rqt * rqt)2376 mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt)
2377 {
2378 	struct mlx5_core_dev *mdev = priv->mdev;
2379 	void *rqtc;
2380 	int inlen;
2381 	int err;
2382 	u32 *in;
2383 	int i;
2384 
2385 	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
2386 	in = kvzalloc(inlen, GFP_KERNEL);
2387 	if (!in)
2388 		return -ENOMEM;
2389 
2390 	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
2391 
2392 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2393 	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
2394 
2395 	for (i = 0; i < sz; i++)
2396 		MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn);
2397 
2398 	err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn);
2399 	if (!err)
2400 		rqt->enabled = true;
2401 
2402 	kvfree(in);
2403 	return err;
2404 }
2405 
mlx5e_destroy_rqt(struct mlx5e_priv * priv,struct mlx5e_rqt * rqt)2406 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt)
2407 {
2408 	rqt->enabled = false;
2409 	mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn);
2410 }
2411 
mlx5e_create_indirect_rqt(struct mlx5e_priv * priv)2412 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv)
2413 {
2414 	struct mlx5e_rqt *rqt = &priv->indir_rqt;
2415 	int err;
2416 
2417 	err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt);
2418 	if (err)
2419 		mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err);
2420 	return err;
2421 }
2422 
mlx5e_create_direct_rqts(struct mlx5e_priv * priv)2423 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv)
2424 {
2425 	struct mlx5e_rqt *rqt;
2426 	int err;
2427 	int ix;
2428 
2429 	for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2430 		rqt = &priv->direct_tir[ix].rqt;
2431 		err = mlx5e_create_rqt(priv, 1 /*size */, rqt);
2432 		if (err)
2433 			goto err_destroy_rqts;
2434 	}
2435 
2436 	return 0;
2437 
2438 err_destroy_rqts:
2439 	mlx5_core_warn(priv->mdev, "create direct rqts failed, %d\n", err);
2440 	for (ix--; ix >= 0; ix--)
2441 		mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt);
2442 
2443 	return err;
2444 }
2445 
mlx5e_destroy_direct_rqts(struct mlx5e_priv * priv)2446 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv)
2447 {
2448 	int i;
2449 
2450 	for (i = 0; i < priv->profile->max_nch(priv->mdev); i++)
2451 		mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt);
2452 }
2453 
mlx5e_rx_hash_fn(int hfunc)2454 static int mlx5e_rx_hash_fn(int hfunc)
2455 {
2456 	return (hfunc == ETH_RSS_HASH_TOP) ?
2457 	       MLX5_RX_HASH_FN_TOEPLITZ :
2458 	       MLX5_RX_HASH_FN_INVERTED_XOR8;
2459 }
2460 
mlx5e_bits_invert(unsigned long a,int size)2461 int mlx5e_bits_invert(unsigned long a, int size)
2462 {
2463 	int inv = 0;
2464 	int i;
2465 
2466 	for (i = 0; i < size; i++)
2467 		inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i;
2468 
2469 	return inv;
2470 }
2471 
mlx5e_fill_rqt_rqns(struct mlx5e_priv * priv,int sz,struct mlx5e_redirect_rqt_param rrp,void * rqtc)2472 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz,
2473 				struct mlx5e_redirect_rqt_param rrp, void *rqtc)
2474 {
2475 	int i;
2476 
2477 	for (i = 0; i < sz; i++) {
2478 		u32 rqn;
2479 
2480 		if (rrp.is_rss) {
2481 			int ix = i;
2482 
2483 			if (rrp.rss.hfunc == ETH_RSS_HASH_XOR)
2484 				ix = mlx5e_bits_invert(i, ilog2(sz));
2485 
2486 			ix = priv->channels.params.indirection_rqt[ix];
2487 			rqn = rrp.rss.channels->c[ix]->rq.rqn;
2488 		} else {
2489 			rqn = rrp.rqn;
2490 		}
2491 		MLX5_SET(rqtc, rqtc, rq_num[i], rqn);
2492 	}
2493 }
2494 
mlx5e_redirect_rqt(struct mlx5e_priv * priv,u32 rqtn,int sz,struct mlx5e_redirect_rqt_param rrp)2495 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz,
2496 		       struct mlx5e_redirect_rqt_param rrp)
2497 {
2498 	struct mlx5_core_dev *mdev = priv->mdev;
2499 	void *rqtc;
2500 	int inlen;
2501 	u32 *in;
2502 	int err;
2503 
2504 	inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz;
2505 	in = kvzalloc(inlen, GFP_KERNEL);
2506 	if (!in)
2507 		return -ENOMEM;
2508 
2509 	rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx);
2510 
2511 	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
2512 	MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1);
2513 	mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc);
2514 	err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen);
2515 
2516 	kvfree(in);
2517 	return err;
2518 }
2519 
mlx5e_get_direct_rqn(struct mlx5e_priv * priv,int ix,struct mlx5e_redirect_rqt_param rrp)2520 static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix,
2521 				struct mlx5e_redirect_rqt_param rrp)
2522 {
2523 	if (!rrp.is_rss)
2524 		return rrp.rqn;
2525 
2526 	if (ix >= rrp.rss.channels->num)
2527 		return priv->drop_rq.rqn;
2528 
2529 	return rrp.rss.channels->c[ix]->rq.rqn;
2530 }
2531 
mlx5e_redirect_rqts(struct mlx5e_priv * priv,struct mlx5e_redirect_rqt_param rrp)2532 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv,
2533 				struct mlx5e_redirect_rqt_param rrp)
2534 {
2535 	u32 rqtn;
2536 	int ix;
2537 
2538 	if (priv->indir_rqt.enabled) {
2539 		/* RSS RQ table */
2540 		rqtn = priv->indir_rqt.rqtn;
2541 		mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp);
2542 	}
2543 
2544 	for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2545 		struct mlx5e_redirect_rqt_param direct_rrp = {
2546 			.is_rss = false,
2547 			{
2548 				.rqn    = mlx5e_get_direct_rqn(priv, ix, rrp)
2549 			},
2550 		};
2551 
2552 		/* Direct RQ Tables */
2553 		if (!priv->direct_tir[ix].rqt.enabled)
2554 			continue;
2555 
2556 		rqtn = priv->direct_tir[ix].rqt.rqtn;
2557 		mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp);
2558 	}
2559 }
2560 
mlx5e_redirect_rqts_to_channels(struct mlx5e_priv * priv,struct mlx5e_channels * chs)2561 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv,
2562 					    struct mlx5e_channels *chs)
2563 {
2564 	struct mlx5e_redirect_rqt_param rrp = {
2565 		.is_rss        = true,
2566 		{
2567 			.rss = {
2568 				.channels  = chs,
2569 				.hfunc     = chs->params.rss_hfunc,
2570 			}
2571 		},
2572 	};
2573 
2574 	mlx5e_redirect_rqts(priv, rrp);
2575 }
2576 
mlx5e_redirect_rqts_to_drop(struct mlx5e_priv * priv)2577 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv)
2578 {
2579 	struct mlx5e_redirect_rqt_param drop_rrp = {
2580 		.is_rss = false,
2581 		{
2582 			.rqn = priv->drop_rq.rqn,
2583 		},
2584 	};
2585 
2586 	mlx5e_redirect_rqts(priv, drop_rrp);
2587 }
2588 
mlx5e_build_tir_ctx_lro(struct mlx5e_params * params,void * tirc)2589 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc)
2590 {
2591 	if (!params->lro_en)
2592 		return;
2593 
2594 #define ROUGH_MAX_L2_L3_HDR_SZ 256
2595 
2596 	MLX5_SET(tirc, tirc, lro_enable_mask,
2597 		 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO |
2598 		 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO);
2599 	MLX5_SET(tirc, tirc, lro_max_ip_payload_size,
2600 		 (params->lro_wqe_sz - ROUGH_MAX_L2_L3_HDR_SZ) >> 8);
2601 	MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout);
2602 }
2603 
mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params * params,enum mlx5e_traffic_types tt,void * tirc,bool inner)2604 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_params *params,
2605 				    enum mlx5e_traffic_types tt,
2606 				    void *tirc, bool inner)
2607 {
2608 	void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) :
2609 			     MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
2610 
2611 #define MLX5_HASH_IP            (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2612 				 MLX5_HASH_FIELD_SEL_DST_IP)
2613 
2614 #define MLX5_HASH_IP_L4PORTS    (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2615 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
2616 				 MLX5_HASH_FIELD_SEL_L4_SPORT |\
2617 				 MLX5_HASH_FIELD_SEL_L4_DPORT)
2618 
2619 #define MLX5_HASH_IP_IPSEC_SPI  (MLX5_HASH_FIELD_SEL_SRC_IP   |\
2620 				 MLX5_HASH_FIELD_SEL_DST_IP   |\
2621 				 MLX5_HASH_FIELD_SEL_IPSEC_SPI)
2622 
2623 	MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(params->rss_hfunc));
2624 	if (params->rss_hfunc == ETH_RSS_HASH_TOP) {
2625 		void *rss_key = MLX5_ADDR_OF(tirc, tirc,
2626 					     rx_hash_toeplitz_key);
2627 		size_t len = MLX5_FLD_SZ_BYTES(tirc,
2628 					       rx_hash_toeplitz_key);
2629 
2630 		MLX5_SET(tirc, tirc, rx_hash_symmetric, 1);
2631 		memcpy(rss_key, params->toeplitz_hash_key, len);
2632 	}
2633 
2634 	switch (tt) {
2635 	case MLX5E_TT_IPV4_TCP:
2636 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2637 			 MLX5_L3_PROT_TYPE_IPV4);
2638 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2639 			 MLX5_L4_PROT_TYPE_TCP);
2640 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2641 			 MLX5_HASH_IP_L4PORTS);
2642 		break;
2643 
2644 	case MLX5E_TT_IPV6_TCP:
2645 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2646 			 MLX5_L3_PROT_TYPE_IPV6);
2647 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2648 			 MLX5_L4_PROT_TYPE_TCP);
2649 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2650 			 MLX5_HASH_IP_L4PORTS);
2651 		break;
2652 
2653 	case MLX5E_TT_IPV4_UDP:
2654 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2655 			 MLX5_L3_PROT_TYPE_IPV4);
2656 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2657 			 MLX5_L4_PROT_TYPE_UDP);
2658 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2659 			 MLX5_HASH_IP_L4PORTS);
2660 		break;
2661 
2662 	case MLX5E_TT_IPV6_UDP:
2663 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2664 			 MLX5_L3_PROT_TYPE_IPV6);
2665 		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
2666 			 MLX5_L4_PROT_TYPE_UDP);
2667 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2668 			 MLX5_HASH_IP_L4PORTS);
2669 		break;
2670 
2671 	case MLX5E_TT_IPV4_IPSEC_AH:
2672 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2673 			 MLX5_L3_PROT_TYPE_IPV4);
2674 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2675 			 MLX5_HASH_IP_IPSEC_SPI);
2676 		break;
2677 
2678 	case MLX5E_TT_IPV6_IPSEC_AH:
2679 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2680 			 MLX5_L3_PROT_TYPE_IPV6);
2681 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2682 			 MLX5_HASH_IP_IPSEC_SPI);
2683 		break;
2684 
2685 	case MLX5E_TT_IPV4_IPSEC_ESP:
2686 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2687 			 MLX5_L3_PROT_TYPE_IPV4);
2688 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2689 			 MLX5_HASH_IP_IPSEC_SPI);
2690 		break;
2691 
2692 	case MLX5E_TT_IPV6_IPSEC_ESP:
2693 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2694 			 MLX5_L3_PROT_TYPE_IPV6);
2695 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2696 			 MLX5_HASH_IP_IPSEC_SPI);
2697 		break;
2698 
2699 	case MLX5E_TT_IPV4:
2700 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2701 			 MLX5_L3_PROT_TYPE_IPV4);
2702 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2703 			 MLX5_HASH_IP);
2704 		break;
2705 
2706 	case MLX5E_TT_IPV6:
2707 		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
2708 			 MLX5_L3_PROT_TYPE_IPV6);
2709 		MLX5_SET(rx_hash_field_select, hfso, selected_fields,
2710 			 MLX5_HASH_IP);
2711 		break;
2712 	default:
2713 		WARN_ONCE(true, "%s: bad traffic type!\n", __func__);
2714 	}
2715 }
2716 
mlx5e_modify_tirs_lro(struct mlx5e_priv * priv)2717 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv)
2718 {
2719 	struct mlx5_core_dev *mdev = priv->mdev;
2720 
2721 	void *in;
2722 	void *tirc;
2723 	int inlen;
2724 	int err;
2725 	int tt;
2726 	int ix;
2727 
2728 	inlen = MLX5_ST_SZ_BYTES(modify_tir_in);
2729 	in = kvzalloc(inlen, GFP_KERNEL);
2730 	if (!in)
2731 		return -ENOMEM;
2732 
2733 	MLX5_SET(modify_tir_in, in, bitmask.lro, 1);
2734 	tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx);
2735 
2736 	mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
2737 
2738 	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
2739 		err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in,
2740 					   inlen);
2741 		if (err)
2742 			goto free_in;
2743 	}
2744 
2745 	for (ix = 0; ix < priv->profile->max_nch(priv->mdev); ix++) {
2746 		err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn,
2747 					   in, inlen);
2748 		if (err)
2749 			goto free_in;
2750 	}
2751 
2752 free_in:
2753 	kvfree(in);
2754 
2755 	return err;
2756 }
2757 
mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv * priv,enum mlx5e_traffic_types tt,u32 * tirc)2758 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv,
2759 					    enum mlx5e_traffic_types tt,
2760 					    u32 *tirc)
2761 {
2762 	MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
2763 
2764 	mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
2765 
2766 	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
2767 	MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
2768 	MLX5_SET(tirc, tirc, tunneled_offload_en, 0x1);
2769 
2770 	mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, true);
2771 }
2772 
mlx5e_set_mtu(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 mtu)2773 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev,
2774 			 struct mlx5e_params *params, u16 mtu)
2775 {
2776 	u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu);
2777 	int err;
2778 
2779 	err = mlx5_set_port_mtu(mdev, hw_mtu, 1);
2780 	if (err)
2781 		return err;
2782 
2783 	/* Update vport context MTU */
2784 	mlx5_modify_nic_vport_mtu(mdev, hw_mtu);
2785 	return 0;
2786 }
2787 
mlx5e_query_mtu(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 * mtu)2788 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev,
2789 			    struct mlx5e_params *params, u16 *mtu)
2790 {
2791 	u16 hw_mtu = 0;
2792 	int err;
2793 
2794 	err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu);
2795 	if (err || !hw_mtu) /* fallback to port oper mtu */
2796 		mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1);
2797 
2798 	*mtu = MLX5E_HW2SW_MTU(params, hw_mtu);
2799 }
2800 
mlx5e_set_dev_port_mtu(struct mlx5e_priv * priv)2801 static int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv)
2802 {
2803 	struct mlx5e_params *params = &priv->channels.params;
2804 	struct net_device *netdev = priv->netdev;
2805 	struct mlx5_core_dev *mdev = priv->mdev;
2806 	u16 mtu;
2807 	int err;
2808 
2809 	err = mlx5e_set_mtu(mdev, params, params->sw_mtu);
2810 	if (err)
2811 		return err;
2812 
2813 	mlx5e_query_mtu(mdev, params, &mtu);
2814 	if (mtu != params->sw_mtu)
2815 		netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n",
2816 			    __func__, mtu, params->sw_mtu);
2817 
2818 	params->sw_mtu = mtu;
2819 	return 0;
2820 }
2821 
mlx5e_netdev_set_tcs(struct net_device * netdev)2822 static void mlx5e_netdev_set_tcs(struct net_device *netdev)
2823 {
2824 	struct mlx5e_priv *priv = netdev_priv(netdev);
2825 	int nch = priv->channels.params.num_channels;
2826 	int ntc = priv->channels.params.num_tc;
2827 	int tc;
2828 
2829 	netdev_reset_tc(netdev);
2830 
2831 	if (ntc == 1)
2832 		return;
2833 
2834 	netdev_set_num_tc(netdev, ntc);
2835 
2836 	/* Map netdev TCs to offset 0
2837 	 * We have our own UP to TXQ mapping for QoS
2838 	 */
2839 	for (tc = 0; tc < ntc; tc++)
2840 		netdev_set_tc_queue(netdev, tc, nch, 0);
2841 }
2842 
mlx5e_build_tc2txq_maps(struct mlx5e_priv * priv)2843 static void mlx5e_build_tc2txq_maps(struct mlx5e_priv *priv)
2844 {
2845 	int max_nch = priv->profile->max_nch(priv->mdev);
2846 	int i, tc;
2847 
2848 	for (i = 0; i < max_nch; i++)
2849 		for (tc = 0; tc < priv->profile->max_tc; tc++)
2850 			priv->channel_tc2txq[i][tc] = i + tc * max_nch;
2851 }
2852 
mlx5e_build_tx2sq_maps(struct mlx5e_priv * priv)2853 static void mlx5e_build_tx2sq_maps(struct mlx5e_priv *priv)
2854 {
2855 	struct mlx5e_channel *c;
2856 	struct mlx5e_txqsq *sq;
2857 	int i, tc;
2858 
2859 	for (i = 0; i < priv->channels.num; i++) {
2860 		c = priv->channels.c[i];
2861 		for (tc = 0; tc < c->num_tc; tc++) {
2862 			sq = &c->sq[tc];
2863 			priv->txq2sq[sq->txq_ix] = sq;
2864 		}
2865 	}
2866 }
2867 
mlx5e_activate_priv_channels(struct mlx5e_priv * priv)2868 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv)
2869 {
2870 	int num_txqs = priv->channels.num * priv->channels.params.num_tc;
2871 	struct net_device *netdev = priv->netdev;
2872 
2873 	mlx5e_netdev_set_tcs(netdev);
2874 	netif_set_real_num_tx_queues(netdev, num_txqs);
2875 	netif_set_real_num_rx_queues(netdev, priv->channels.num);
2876 
2877 	mlx5e_build_tx2sq_maps(priv);
2878 	mlx5e_activate_channels(&priv->channels);
2879 	netif_tx_start_all_queues(priv->netdev);
2880 
2881 	if (MLX5_ESWITCH_MANAGER(priv->mdev))
2882 		mlx5e_add_sqs_fwd_rules(priv);
2883 
2884 	mlx5e_wait_channels_min_rx_wqes(&priv->channels);
2885 	mlx5e_redirect_rqts_to_channels(priv, &priv->channels);
2886 }
2887 
mlx5e_deactivate_priv_channels(struct mlx5e_priv * priv)2888 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv)
2889 {
2890 	mlx5e_redirect_rqts_to_drop(priv);
2891 
2892 	if (MLX5_ESWITCH_MANAGER(priv->mdev))
2893 		mlx5e_remove_sqs_fwd_rules(priv);
2894 
2895 	/* FIXME: This is a W/A only for tx timeout watch dog false alarm when
2896 	 * polling for inactive tx queues.
2897 	 */
2898 	netif_tx_stop_all_queues(priv->netdev);
2899 	netif_tx_disable(priv->netdev);
2900 	mlx5e_deactivate_channels(&priv->channels);
2901 }
2902 
mlx5e_switch_priv_channels(struct mlx5e_priv * priv,struct mlx5e_channels * new_chs,mlx5e_fp_hw_modify hw_modify)2903 void mlx5e_switch_priv_channels(struct mlx5e_priv *priv,
2904 				struct mlx5e_channels *new_chs,
2905 				mlx5e_fp_hw_modify hw_modify)
2906 {
2907 	struct net_device *netdev = priv->netdev;
2908 	int new_num_txqs;
2909 	int carrier_ok;
2910 	new_num_txqs = new_chs->num * new_chs->params.num_tc;
2911 
2912 	carrier_ok = netif_carrier_ok(netdev);
2913 	netif_carrier_off(netdev);
2914 
2915 	if (new_num_txqs < netdev->real_num_tx_queues)
2916 		netif_set_real_num_tx_queues(netdev, new_num_txqs);
2917 
2918 	mlx5e_deactivate_priv_channels(priv);
2919 	mlx5e_close_channels(&priv->channels);
2920 
2921 	priv->channels = *new_chs;
2922 
2923 	/* New channels are ready to roll, modify HW settings if needed */
2924 	if (hw_modify)
2925 		hw_modify(priv);
2926 
2927 	mlx5e_refresh_tirs(priv, false);
2928 	mlx5e_activate_priv_channels(priv);
2929 
2930 	/* return carrier back if needed */
2931 	if (carrier_ok)
2932 		netif_carrier_on(netdev);
2933 }
2934 
mlx5e_timestamp_init(struct mlx5e_priv * priv)2935 void mlx5e_timestamp_init(struct mlx5e_priv *priv)
2936 {
2937 	priv->tstamp.tx_type   = HWTSTAMP_TX_OFF;
2938 	priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE;
2939 }
2940 
mlx5e_open_locked(struct net_device * netdev)2941 int mlx5e_open_locked(struct net_device *netdev)
2942 {
2943 	struct mlx5e_priv *priv = netdev_priv(netdev);
2944 	int err;
2945 
2946 	set_bit(MLX5E_STATE_OPENED, &priv->state);
2947 
2948 	err = mlx5e_open_channels(priv, &priv->channels);
2949 	if (err)
2950 		goto err_clear_state_opened_flag;
2951 
2952 	mlx5e_refresh_tirs(priv, false);
2953 	mlx5e_activate_priv_channels(priv);
2954 	if (priv->profile->update_carrier)
2955 		priv->profile->update_carrier(priv);
2956 
2957 	if (priv->profile->update_stats)
2958 		queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
2959 
2960 	return 0;
2961 
2962 err_clear_state_opened_flag:
2963 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
2964 	return err;
2965 }
2966 
mlx5e_open(struct net_device * netdev)2967 int mlx5e_open(struct net_device *netdev)
2968 {
2969 	struct mlx5e_priv *priv = netdev_priv(netdev);
2970 	int err;
2971 
2972 	mutex_lock(&priv->state_lock);
2973 	err = mlx5e_open_locked(netdev);
2974 	if (!err)
2975 		mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP);
2976 	mutex_unlock(&priv->state_lock);
2977 
2978 	if (mlx5_vxlan_allowed(priv->mdev->vxlan))
2979 		udp_tunnel_get_rx_info(netdev);
2980 
2981 	return err;
2982 }
2983 
mlx5e_close_locked(struct net_device * netdev)2984 int mlx5e_close_locked(struct net_device *netdev)
2985 {
2986 	struct mlx5e_priv *priv = netdev_priv(netdev);
2987 
2988 	/* May already be CLOSED in case a previous configuration operation
2989 	 * (e.g RX/TX queue size change) that involves close&open failed.
2990 	 */
2991 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
2992 		return 0;
2993 
2994 	clear_bit(MLX5E_STATE_OPENED, &priv->state);
2995 
2996 	netif_carrier_off(priv->netdev);
2997 	mlx5e_deactivate_priv_channels(priv);
2998 	mlx5e_close_channels(&priv->channels);
2999 
3000 	return 0;
3001 }
3002 
mlx5e_close(struct net_device * netdev)3003 int mlx5e_close(struct net_device *netdev)
3004 {
3005 	struct mlx5e_priv *priv = netdev_priv(netdev);
3006 	int err;
3007 
3008 	if (!netif_device_present(netdev))
3009 		return -ENODEV;
3010 
3011 	mutex_lock(&priv->state_lock);
3012 	mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN);
3013 	err = mlx5e_close_locked(netdev);
3014 	mutex_unlock(&priv->state_lock);
3015 
3016 	return err;
3017 }
3018 
mlx5e_alloc_drop_rq(struct mlx5_core_dev * mdev,struct mlx5e_rq * rq,struct mlx5e_rq_param * param)3019 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev,
3020 			       struct mlx5e_rq *rq,
3021 			       struct mlx5e_rq_param *param)
3022 {
3023 	void *rqc = param->rqc;
3024 	void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq);
3025 	int err;
3026 
3027 	param->wq.db_numa_node = param->wq.buf_numa_node;
3028 
3029 	err = mlx5_wq_cyc_create(mdev, &param->wq, rqc_wq, &rq->wqe.wq,
3030 				 &rq->wq_ctrl);
3031 	if (err)
3032 		return err;
3033 
3034 	/* Mark as unused given "Drop-RQ" packets never reach XDP */
3035 	xdp_rxq_info_unused(&rq->xdp_rxq);
3036 
3037 	rq->mdev = mdev;
3038 
3039 	return 0;
3040 }
3041 
mlx5e_alloc_drop_cq(struct mlx5_core_dev * mdev,struct mlx5e_cq * cq,struct mlx5e_cq_param * param)3042 static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev,
3043 			       struct mlx5e_cq *cq,
3044 			       struct mlx5e_cq_param *param)
3045 {
3046 	param->wq.buf_numa_node = dev_to_node(&mdev->pdev->dev);
3047 	param->wq.db_numa_node  = dev_to_node(&mdev->pdev->dev);
3048 
3049 	return mlx5e_alloc_cq_common(mdev, param, cq);
3050 }
3051 
mlx5e_open_drop_rq(struct mlx5e_priv * priv,struct mlx5e_rq * drop_rq)3052 static int mlx5e_open_drop_rq(struct mlx5e_priv *priv,
3053 			      struct mlx5e_rq *drop_rq)
3054 {
3055 	struct mlx5_core_dev *mdev = priv->mdev;
3056 	struct mlx5e_cq_param cq_param = {};
3057 	struct mlx5e_rq_param rq_param = {};
3058 	struct mlx5e_cq *cq = &drop_rq->cq;
3059 	int err;
3060 
3061 	mlx5e_build_drop_rq_param(priv, &rq_param);
3062 
3063 	err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param);
3064 	if (err)
3065 		return err;
3066 
3067 	err = mlx5e_create_cq(cq, &cq_param);
3068 	if (err)
3069 		goto err_free_cq;
3070 
3071 	err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param);
3072 	if (err)
3073 		goto err_destroy_cq;
3074 
3075 	err = mlx5e_create_rq(drop_rq, &rq_param);
3076 	if (err)
3077 		goto err_free_rq;
3078 
3079 	err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY);
3080 	if (err)
3081 		mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err);
3082 
3083 	return 0;
3084 
3085 err_free_rq:
3086 	mlx5e_free_rq(drop_rq);
3087 
3088 err_destroy_cq:
3089 	mlx5e_destroy_cq(cq);
3090 
3091 err_free_cq:
3092 	mlx5e_free_cq(cq);
3093 
3094 	return err;
3095 }
3096 
mlx5e_close_drop_rq(struct mlx5e_rq * drop_rq)3097 static void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq)
3098 {
3099 	mlx5e_destroy_rq(drop_rq);
3100 	mlx5e_free_rq(drop_rq);
3101 	mlx5e_destroy_cq(&drop_rq->cq);
3102 	mlx5e_free_cq(&drop_rq->cq);
3103 }
3104 
mlx5e_create_tis(struct mlx5_core_dev * mdev,int tc,u32 underlay_qpn,u32 * tisn)3105 int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc,
3106 		     u32 underlay_qpn, u32 *tisn)
3107 {
3108 	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0};
3109 	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
3110 
3111 	MLX5_SET(tisc, tisc, prio, tc << 1);
3112 	MLX5_SET(tisc, tisc, underlay_qpn, underlay_qpn);
3113 	MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn);
3114 
3115 	if (mlx5_lag_is_lacp_owner(mdev))
3116 		MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
3117 
3118 	return mlx5_core_create_tis(mdev, in, sizeof(in), tisn);
3119 }
3120 
mlx5e_destroy_tis(struct mlx5_core_dev * mdev,u32 tisn)3121 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn)
3122 {
3123 	mlx5_core_destroy_tis(mdev, tisn);
3124 }
3125 
mlx5e_create_tises(struct mlx5e_priv * priv)3126 int mlx5e_create_tises(struct mlx5e_priv *priv)
3127 {
3128 	int err;
3129 	int tc;
3130 
3131 	for (tc = 0; tc < priv->profile->max_tc; tc++) {
3132 		err = mlx5e_create_tis(priv->mdev, tc, 0, &priv->tisn[tc]);
3133 		if (err)
3134 			goto err_close_tises;
3135 	}
3136 
3137 	return 0;
3138 
3139 err_close_tises:
3140 	for (tc--; tc >= 0; tc--)
3141 		mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
3142 
3143 	return err;
3144 }
3145 
mlx5e_cleanup_nic_tx(struct mlx5e_priv * priv)3146 void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv)
3147 {
3148 	int tc;
3149 
3150 	for (tc = 0; tc < priv->profile->max_tc; tc++)
3151 		mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]);
3152 }
3153 
mlx5e_build_indir_tir_ctx(struct mlx5e_priv * priv,enum mlx5e_traffic_types tt,u32 * tirc)3154 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv,
3155 				      enum mlx5e_traffic_types tt,
3156 				      u32 *tirc)
3157 {
3158 	MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3159 
3160 	mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
3161 
3162 	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
3163 	MLX5_SET(tirc, tirc, indirect_table, priv->indir_rqt.rqtn);
3164 	mlx5e_build_indir_tir_ctx_hash(&priv->channels.params, tt, tirc, false);
3165 }
3166 
mlx5e_build_direct_tir_ctx(struct mlx5e_priv * priv,u32 rqtn,u32 * tirc)3167 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc)
3168 {
3169 	MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn);
3170 
3171 	mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc);
3172 
3173 	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT);
3174 	MLX5_SET(tirc, tirc, indirect_table, rqtn);
3175 	MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8);
3176 }
3177 
mlx5e_create_indirect_tirs(struct mlx5e_priv * priv)3178 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv)
3179 {
3180 	struct mlx5e_tir *tir;
3181 	void *tirc;
3182 	int inlen;
3183 	int i = 0;
3184 	int err;
3185 	u32 *in;
3186 	int tt;
3187 
3188 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3189 	in = kvzalloc(inlen, GFP_KERNEL);
3190 	if (!in)
3191 		return -ENOMEM;
3192 
3193 	for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) {
3194 		memset(in, 0, inlen);
3195 		tir = &priv->indir_tir[tt];
3196 		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3197 		mlx5e_build_indir_tir_ctx(priv, tt, tirc);
3198 		err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
3199 		if (err) {
3200 			mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err);
3201 			goto err_destroy_inner_tirs;
3202 		}
3203 	}
3204 
3205 	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
3206 		goto out;
3207 
3208 	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) {
3209 		memset(in, 0, inlen);
3210 		tir = &priv->inner_indir_tir[i];
3211 		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3212 		mlx5e_build_inner_indir_tir_ctx(priv, i, tirc);
3213 		err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
3214 		if (err) {
3215 			mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err);
3216 			goto err_destroy_inner_tirs;
3217 		}
3218 	}
3219 
3220 out:
3221 	kvfree(in);
3222 
3223 	return 0;
3224 
3225 err_destroy_inner_tirs:
3226 	for (i--; i >= 0; i--)
3227 		mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3228 
3229 	for (tt--; tt >= 0; tt--)
3230 		mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]);
3231 
3232 	kvfree(in);
3233 
3234 	return err;
3235 }
3236 
mlx5e_create_direct_tirs(struct mlx5e_priv * priv)3237 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv)
3238 {
3239 	int nch = priv->profile->max_nch(priv->mdev);
3240 	struct mlx5e_tir *tir;
3241 	void *tirc;
3242 	int inlen;
3243 	int err;
3244 	u32 *in;
3245 	int ix;
3246 
3247 	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
3248 	in = kvzalloc(inlen, GFP_KERNEL);
3249 	if (!in)
3250 		return -ENOMEM;
3251 
3252 	for (ix = 0; ix < nch; ix++) {
3253 		memset(in, 0, inlen);
3254 		tir = &priv->direct_tir[ix];
3255 		tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
3256 		mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc);
3257 		err = mlx5e_create_tir(priv->mdev, tir, in, inlen);
3258 		if (err)
3259 			goto err_destroy_ch_tirs;
3260 	}
3261 
3262 	kvfree(in);
3263 
3264 	return 0;
3265 
3266 err_destroy_ch_tirs:
3267 	mlx5_core_warn(priv->mdev, "create direct tirs failed, %d\n", err);
3268 	for (ix--; ix >= 0; ix--)
3269 		mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]);
3270 
3271 	kvfree(in);
3272 
3273 	return err;
3274 }
3275 
mlx5e_destroy_indirect_tirs(struct mlx5e_priv * priv)3276 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv)
3277 {
3278 	int i;
3279 
3280 	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3281 		mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]);
3282 
3283 	if (!mlx5e_tunnel_inner_ft_supported(priv->mdev))
3284 		return;
3285 
3286 	for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++)
3287 		mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]);
3288 }
3289 
mlx5e_destroy_direct_tirs(struct mlx5e_priv * priv)3290 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv)
3291 {
3292 	int nch = priv->profile->max_nch(priv->mdev);
3293 	int i;
3294 
3295 	for (i = 0; i < nch; i++)
3296 		mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]);
3297 }
3298 
mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels * chs,bool enable)3299 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable)
3300 {
3301 	int err = 0;
3302 	int i;
3303 
3304 	for (i = 0; i < chs->num; i++) {
3305 		err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable);
3306 		if (err)
3307 			return err;
3308 	}
3309 
3310 	return 0;
3311 }
3312 
mlx5e_modify_channels_vsd(struct mlx5e_channels * chs,bool vsd)3313 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd)
3314 {
3315 	int err = 0;
3316 	int i;
3317 
3318 	for (i = 0; i < chs->num; i++) {
3319 		err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd);
3320 		if (err)
3321 			return err;
3322 	}
3323 
3324 	return 0;
3325 }
3326 
mlx5e_setup_tc_mqprio(struct net_device * netdev,struct tc_mqprio_qopt * mqprio)3327 static int mlx5e_setup_tc_mqprio(struct net_device *netdev,
3328 				 struct tc_mqprio_qopt *mqprio)
3329 {
3330 	struct mlx5e_priv *priv = netdev_priv(netdev);
3331 	struct mlx5e_channels new_channels = {};
3332 	u8 tc = mqprio->num_tc;
3333 	int err = 0;
3334 
3335 	mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
3336 
3337 	if (tc && tc != MLX5E_MAX_NUM_TC)
3338 		return -EINVAL;
3339 
3340 	mutex_lock(&priv->state_lock);
3341 
3342 	new_channels.params = priv->channels.params;
3343 	new_channels.params.num_tc = tc ? tc : 1;
3344 
3345 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) {
3346 		priv->channels.params = new_channels.params;
3347 		goto out;
3348 	}
3349 
3350 	err = mlx5e_open_channels(priv, &new_channels);
3351 	if (err)
3352 		goto out;
3353 
3354 	priv->max_opened_tc = max_t(u8, priv->max_opened_tc,
3355 				    new_channels.params.num_tc);
3356 	mlx5e_switch_priv_channels(priv, &new_channels, NULL);
3357 out:
3358 	mutex_unlock(&priv->state_lock);
3359 	return err;
3360 }
3361 
3362 #ifdef CONFIG_MLX5_ESWITCH
mlx5e_setup_tc_cls_flower(struct mlx5e_priv * priv,struct tc_cls_flower_offload * cls_flower,int flags)3363 static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv,
3364 				     struct tc_cls_flower_offload *cls_flower,
3365 				     int flags)
3366 {
3367 	switch (cls_flower->command) {
3368 	case TC_CLSFLOWER_REPLACE:
3369 		return mlx5e_configure_flower(priv, cls_flower, flags);
3370 	case TC_CLSFLOWER_DESTROY:
3371 		return mlx5e_delete_flower(priv, cls_flower, flags);
3372 	case TC_CLSFLOWER_STATS:
3373 		return mlx5e_stats_flower(priv, cls_flower, flags);
3374 	default:
3375 		return -EOPNOTSUPP;
3376 	}
3377 }
3378 
mlx5e_setup_tc_block_cb(enum tc_setup_type type,void * type_data,void * cb_priv)3379 static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
3380 				   void *cb_priv)
3381 {
3382 	struct mlx5e_priv *priv = cb_priv;
3383 
3384 	if (!tc_cls_can_offload_and_chain0(priv->netdev, type_data))
3385 		return -EOPNOTSUPP;
3386 
3387 	switch (type) {
3388 	case TC_SETUP_CLSFLOWER:
3389 		return mlx5e_setup_tc_cls_flower(priv, type_data, MLX5E_TC_INGRESS);
3390 	default:
3391 		return -EOPNOTSUPP;
3392 	}
3393 }
3394 
mlx5e_setup_tc_block(struct net_device * dev,struct tc_block_offload * f)3395 static int mlx5e_setup_tc_block(struct net_device *dev,
3396 				struct tc_block_offload *f)
3397 {
3398 	struct mlx5e_priv *priv = netdev_priv(dev);
3399 
3400 	if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS)
3401 		return -EOPNOTSUPP;
3402 
3403 	switch (f->command) {
3404 	case TC_BLOCK_BIND:
3405 		return tcf_block_cb_register(f->block, mlx5e_setup_tc_block_cb,
3406 					     priv, priv, f->extack);
3407 	case TC_BLOCK_UNBIND:
3408 		tcf_block_cb_unregister(f->block, mlx5e_setup_tc_block_cb,
3409 					priv);
3410 		return 0;
3411 	default:
3412 		return -EOPNOTSUPP;
3413 	}
3414 }
3415 #endif
3416 
mlx5e_setup_tc(struct net_device * dev,enum tc_setup_type type,void * type_data)3417 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
3418 			  void *type_data)
3419 {
3420 	switch (type) {
3421 #ifdef CONFIG_MLX5_ESWITCH
3422 	case TC_SETUP_BLOCK:
3423 		return mlx5e_setup_tc_block(dev, type_data);
3424 #endif
3425 	case TC_SETUP_QDISC_MQPRIO:
3426 		return mlx5e_setup_tc_mqprio(dev, type_data);
3427 	default:
3428 		return -EOPNOTSUPP;
3429 	}
3430 }
3431 
3432 static void
mlx5e_get_stats(struct net_device * dev,struct rtnl_link_stats64 * stats)3433 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
3434 {
3435 	struct mlx5e_priv *priv = netdev_priv(dev);
3436 	struct mlx5e_sw_stats *sstats = &priv->stats.sw;
3437 	struct mlx5e_vport_stats *vstats = &priv->stats.vport;
3438 	struct mlx5e_pport_stats *pstats = &priv->stats.pport;
3439 
3440 	/* update HW stats in background for next time */
3441 	queue_delayed_work(priv->wq, &priv->update_stats_work, 0);
3442 
3443 	if (mlx5e_is_uplink_rep(priv)) {
3444 		stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok);
3445 		stats->rx_bytes   = PPORT_802_3_GET(pstats, a_octets_received_ok);
3446 		stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
3447 		stats->tx_bytes   = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
3448 	} else {
3449 		mlx5e_grp_sw_update_stats(priv);
3450 		stats->rx_packets = sstats->rx_packets;
3451 		stats->rx_bytes   = sstats->rx_bytes;
3452 		stats->tx_packets = sstats->tx_packets;
3453 		stats->tx_bytes   = sstats->tx_bytes;
3454 		stats->tx_dropped = sstats->tx_queue_dropped;
3455 	}
3456 
3457 	stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
3458 
3459 	stats->rx_length_errors =
3460 		PPORT_802_3_GET(pstats, a_in_range_length_errors) +
3461 		PPORT_802_3_GET(pstats, a_out_of_range_length_field) +
3462 		PPORT_802_3_GET(pstats, a_frame_too_long_errors);
3463 	stats->rx_crc_errors =
3464 		PPORT_802_3_GET(pstats, a_frame_check_sequence_errors);
3465 	stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors);
3466 	stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards);
3467 	stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors +
3468 			   stats->rx_frame_errors;
3469 	stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors;
3470 
3471 	/* vport multicast also counts packets that are dropped due to steering
3472 	 * or rx out of buffer
3473 	 */
3474 	stats->multicast =
3475 		VPORT_COUNTER_GET(vstats, received_eth_multicast.packets);
3476 }
3477 
mlx5e_set_rx_mode(struct net_device * dev)3478 static void mlx5e_set_rx_mode(struct net_device *dev)
3479 {
3480 	struct mlx5e_priv *priv = netdev_priv(dev);
3481 
3482 	queue_work(priv->wq, &priv->set_rx_mode_work);
3483 }
3484 
mlx5e_set_mac(struct net_device * netdev,void * addr)3485 static int mlx5e_set_mac(struct net_device *netdev, void *addr)
3486 {
3487 	struct mlx5e_priv *priv = netdev_priv(netdev);
3488 	struct sockaddr *saddr = addr;
3489 
3490 	if (!is_valid_ether_addr(saddr->sa_data))
3491 		return -EADDRNOTAVAIL;
3492 
3493 	netif_addr_lock_bh(netdev);
3494 	ether_addr_copy(netdev->dev_addr, saddr->sa_data);
3495 	netif_addr_unlock_bh(netdev);
3496 
3497 	queue_work(priv->wq, &priv->set_rx_mode_work);
3498 
3499 	return 0;
3500 }
3501 
3502 #define MLX5E_SET_FEATURE(features, feature, enable)	\
3503 	do {						\
3504 		if (enable)				\
3505 			*features |= feature;		\
3506 		else					\
3507 			*features &= ~feature;		\
3508 	} while (0)
3509 
3510 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable);
3511 
set_feature_lro(struct net_device * netdev,bool enable)3512 static int set_feature_lro(struct net_device *netdev, bool enable)
3513 {
3514 	struct mlx5e_priv *priv = netdev_priv(netdev);
3515 	struct mlx5_core_dev *mdev = priv->mdev;
3516 	struct mlx5e_channels new_channels = {};
3517 	struct mlx5e_params *old_params;
3518 	int err = 0;
3519 	bool reset;
3520 
3521 	mutex_lock(&priv->state_lock);
3522 
3523 	old_params = &priv->channels.params;
3524 	if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3525 		netdev_warn(netdev, "can't set LRO with legacy RQ\n");
3526 		err = -EINVAL;
3527 		goto out;
3528 	}
3529 
3530 	reset = test_bit(MLX5E_STATE_OPENED, &priv->state);
3531 
3532 	new_channels.params = *old_params;
3533 	new_channels.params.lro_en = enable;
3534 
3535 	if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) {
3536 		if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params) ==
3537 		    mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params))
3538 			reset = false;
3539 	}
3540 
3541 	if (!reset) {
3542 		*old_params = new_channels.params;
3543 		err = mlx5e_modify_tirs_lro(priv);
3544 		goto out;
3545 	}
3546 
3547 	err = mlx5e_open_channels(priv, &new_channels);
3548 	if (err)
3549 		goto out;
3550 
3551 	mlx5e_switch_priv_channels(priv, &new_channels, mlx5e_modify_tirs_lro);
3552 out:
3553 	mutex_unlock(&priv->state_lock);
3554 	return err;
3555 }
3556 
set_feature_cvlan_filter(struct net_device * netdev,bool enable)3557 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable)
3558 {
3559 	struct mlx5e_priv *priv = netdev_priv(netdev);
3560 
3561 	if (enable)
3562 		mlx5e_enable_cvlan_filter(priv);
3563 	else
3564 		mlx5e_disable_cvlan_filter(priv);
3565 
3566 	return 0;
3567 }
3568 
set_feature_tc_num_filters(struct net_device * netdev,bool enable)3569 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable)
3570 {
3571 	struct mlx5e_priv *priv = netdev_priv(netdev);
3572 
3573 	if (!enable && mlx5e_tc_num_filters(priv)) {
3574 		netdev_err(netdev,
3575 			   "Active offloaded tc filters, can't turn hw_tc_offload off\n");
3576 		return -EINVAL;
3577 	}
3578 
3579 	return 0;
3580 }
3581 
set_feature_rx_all(struct net_device * netdev,bool enable)3582 static int set_feature_rx_all(struct net_device *netdev, bool enable)
3583 {
3584 	struct mlx5e_priv *priv = netdev_priv(netdev);
3585 	struct mlx5_core_dev *mdev = priv->mdev;
3586 
3587 	return mlx5_set_port_fcs(mdev, !enable);
3588 }
3589 
set_feature_rx_fcs(struct net_device * netdev,bool enable)3590 static int set_feature_rx_fcs(struct net_device *netdev, bool enable)
3591 {
3592 	struct mlx5e_priv *priv = netdev_priv(netdev);
3593 	int err;
3594 
3595 	mutex_lock(&priv->state_lock);
3596 
3597 	priv->channels.params.scatter_fcs_en = enable;
3598 	err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable);
3599 	if (err)
3600 		priv->channels.params.scatter_fcs_en = !enable;
3601 
3602 	mutex_unlock(&priv->state_lock);
3603 
3604 	return err;
3605 }
3606 
set_feature_rx_vlan(struct net_device * netdev,bool enable)3607 static int set_feature_rx_vlan(struct net_device *netdev, bool enable)
3608 {
3609 	struct mlx5e_priv *priv = netdev_priv(netdev);
3610 	int err = 0;
3611 
3612 	mutex_lock(&priv->state_lock);
3613 
3614 	priv->channels.params.vlan_strip_disable = !enable;
3615 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
3616 		goto unlock;
3617 
3618 	err = mlx5e_modify_channels_vsd(&priv->channels, !enable);
3619 	if (err)
3620 		priv->channels.params.vlan_strip_disable = enable;
3621 
3622 unlock:
3623 	mutex_unlock(&priv->state_lock);
3624 
3625 	return err;
3626 }
3627 
3628 #ifdef CONFIG_MLX5_EN_ARFS
set_feature_arfs(struct net_device * netdev,bool enable)3629 static int set_feature_arfs(struct net_device *netdev, bool enable)
3630 {
3631 	struct mlx5e_priv *priv = netdev_priv(netdev);
3632 	int err;
3633 
3634 	if (enable)
3635 		err = mlx5e_arfs_enable(priv);
3636 	else
3637 		err = mlx5e_arfs_disable(priv);
3638 
3639 	return err;
3640 }
3641 #endif
3642 
mlx5e_handle_feature(struct net_device * netdev,netdev_features_t * features,netdev_features_t wanted_features,netdev_features_t feature,mlx5e_feature_handler feature_handler)3643 static int mlx5e_handle_feature(struct net_device *netdev,
3644 				netdev_features_t *features,
3645 				netdev_features_t wanted_features,
3646 				netdev_features_t feature,
3647 				mlx5e_feature_handler feature_handler)
3648 {
3649 	netdev_features_t changes = wanted_features ^ netdev->features;
3650 	bool enable = !!(wanted_features & feature);
3651 	int err;
3652 
3653 	if (!(changes & feature))
3654 		return 0;
3655 
3656 	err = feature_handler(netdev, enable);
3657 	if (err) {
3658 		netdev_err(netdev, "%s feature %pNF failed, err %d\n",
3659 			   enable ? "Enable" : "Disable", &feature, err);
3660 		return err;
3661 	}
3662 
3663 	MLX5E_SET_FEATURE(features, feature, enable);
3664 	return 0;
3665 }
3666 
mlx5e_set_features(struct net_device * netdev,netdev_features_t features)3667 static int mlx5e_set_features(struct net_device *netdev,
3668 			      netdev_features_t features)
3669 {
3670 	netdev_features_t oper_features = netdev->features;
3671 	int err = 0;
3672 
3673 #define MLX5E_HANDLE_FEATURE(feature, handler) \
3674 	mlx5e_handle_feature(netdev, &oper_features, features, feature, handler)
3675 
3676 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro);
3677 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER,
3678 				    set_feature_cvlan_filter);
3679 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters);
3680 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all);
3681 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs);
3682 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan);
3683 #ifdef CONFIG_MLX5_EN_ARFS
3684 	err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs);
3685 #endif
3686 
3687 	if (err) {
3688 		netdev->features = oper_features;
3689 		return -EINVAL;
3690 	}
3691 
3692 	return 0;
3693 }
3694 
mlx5e_fix_features(struct net_device * netdev,netdev_features_t features)3695 static netdev_features_t mlx5e_fix_features(struct net_device *netdev,
3696 					    netdev_features_t features)
3697 {
3698 	struct mlx5e_priv *priv = netdev_priv(netdev);
3699 	struct mlx5e_params *params;
3700 
3701 	mutex_lock(&priv->state_lock);
3702 	params = &priv->channels.params;
3703 	if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) {
3704 		/* HW strips the outer C-tag header, this is a problem
3705 		 * for S-tag traffic.
3706 		 */
3707 		features &= ~NETIF_F_HW_VLAN_CTAG_RX;
3708 		if (!params->vlan_strip_disable)
3709 			netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n");
3710 	}
3711 	if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) {
3712 		features &= ~NETIF_F_LRO;
3713 		if (params->lro_en)
3714 			netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n");
3715 	}
3716 
3717 	mutex_unlock(&priv->state_lock);
3718 
3719 	return features;
3720 }
3721 
mlx5e_change_mtu(struct net_device * netdev,int new_mtu,change_hw_mtu_cb set_mtu_cb)3722 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu,
3723 		     change_hw_mtu_cb set_mtu_cb)
3724 {
3725 	struct mlx5e_priv *priv = netdev_priv(netdev);
3726 	struct mlx5e_channels new_channels = {};
3727 	struct mlx5e_params *params;
3728 	int err = 0;
3729 	bool reset;
3730 
3731 	mutex_lock(&priv->state_lock);
3732 
3733 	params = &priv->channels.params;
3734 
3735 	reset = !params->lro_en;
3736 	reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state);
3737 
3738 	new_channels.params = *params;
3739 	new_channels.params.sw_mtu = new_mtu;
3740 
3741 	if (params->xdp_prog &&
3742 	    !mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
3743 		netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n",
3744 			   new_mtu, MLX5E_XDP_MAX_MTU);
3745 		err = -EINVAL;
3746 		goto out;
3747 	}
3748 
3749 	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) {
3750 		u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params);
3751 		u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params);
3752 
3753 		reset = reset && (ppw_old != ppw_new);
3754 	}
3755 
3756 	if (!reset) {
3757 		params->sw_mtu = new_mtu;
3758 		if (set_mtu_cb)
3759 			set_mtu_cb(priv);
3760 		netdev->mtu = params->sw_mtu;
3761 		goto out;
3762 	}
3763 
3764 	err = mlx5e_open_channels(priv, &new_channels);
3765 	if (err)
3766 		goto out;
3767 
3768 	mlx5e_switch_priv_channels(priv, &new_channels, set_mtu_cb);
3769 	netdev->mtu = new_channels.params.sw_mtu;
3770 
3771 out:
3772 	mutex_unlock(&priv->state_lock);
3773 	return err;
3774 }
3775 
mlx5e_change_nic_mtu(struct net_device * netdev,int new_mtu)3776 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu)
3777 {
3778 	return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu);
3779 }
3780 
mlx5e_hwstamp_set(struct mlx5e_priv * priv,struct ifreq * ifr)3781 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr)
3782 {
3783 	struct hwtstamp_config config;
3784 	int err;
3785 
3786 	if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) ||
3787 	    (mlx5_clock_get_ptp_index(priv->mdev) == -1))
3788 		return -EOPNOTSUPP;
3789 
3790 	if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
3791 		return -EFAULT;
3792 
3793 	/* TX HW timestamp */
3794 	switch (config.tx_type) {
3795 	case HWTSTAMP_TX_OFF:
3796 	case HWTSTAMP_TX_ON:
3797 		break;
3798 	default:
3799 		return -ERANGE;
3800 	}
3801 
3802 	mutex_lock(&priv->state_lock);
3803 	/* RX HW timestamp */
3804 	switch (config.rx_filter) {
3805 	case HWTSTAMP_FILTER_NONE:
3806 		/* Reset CQE compression to Admin default */
3807 		mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def);
3808 		break;
3809 	case HWTSTAMP_FILTER_ALL:
3810 	case HWTSTAMP_FILTER_SOME:
3811 	case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
3812 	case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
3813 	case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
3814 	case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
3815 	case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
3816 	case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
3817 	case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
3818 	case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
3819 	case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
3820 	case HWTSTAMP_FILTER_PTP_V2_EVENT:
3821 	case HWTSTAMP_FILTER_PTP_V2_SYNC:
3822 	case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
3823 	case HWTSTAMP_FILTER_NTP_ALL:
3824 		/* Disable CQE compression */
3825 		netdev_warn(priv->netdev, "Disabling cqe compression");
3826 		err = mlx5e_modify_rx_cqe_compression_locked(priv, false);
3827 		if (err) {
3828 			netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err);
3829 			mutex_unlock(&priv->state_lock);
3830 			return err;
3831 		}
3832 		config.rx_filter = HWTSTAMP_FILTER_ALL;
3833 		break;
3834 	default:
3835 		mutex_unlock(&priv->state_lock);
3836 		return -ERANGE;
3837 	}
3838 
3839 	memcpy(&priv->tstamp, &config, sizeof(config));
3840 	mutex_unlock(&priv->state_lock);
3841 
3842 	return copy_to_user(ifr->ifr_data, &config,
3843 			    sizeof(config)) ? -EFAULT : 0;
3844 }
3845 
mlx5e_hwstamp_get(struct mlx5e_priv * priv,struct ifreq * ifr)3846 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr)
3847 {
3848 	struct hwtstamp_config *cfg = &priv->tstamp;
3849 
3850 	if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz))
3851 		return -EOPNOTSUPP;
3852 
3853 	return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0;
3854 }
3855 
mlx5e_ioctl(struct net_device * dev,struct ifreq * ifr,int cmd)3856 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
3857 {
3858 	struct mlx5e_priv *priv = netdev_priv(dev);
3859 
3860 	switch (cmd) {
3861 	case SIOCSHWTSTAMP:
3862 		return mlx5e_hwstamp_set(priv, ifr);
3863 	case SIOCGHWTSTAMP:
3864 		return mlx5e_hwstamp_get(priv, ifr);
3865 	default:
3866 		return -EOPNOTSUPP;
3867 	}
3868 }
3869 
3870 #ifdef CONFIG_MLX5_ESWITCH
mlx5e_set_vf_mac(struct net_device * dev,int vf,u8 * mac)3871 static int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac)
3872 {
3873 	struct mlx5e_priv *priv = netdev_priv(dev);
3874 	struct mlx5_core_dev *mdev = priv->mdev;
3875 
3876 	return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac);
3877 }
3878 
mlx5e_set_vf_vlan(struct net_device * dev,int vf,u16 vlan,u8 qos,__be16 vlan_proto)3879 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos,
3880 			     __be16 vlan_proto)
3881 {
3882 	struct mlx5e_priv *priv = netdev_priv(dev);
3883 	struct mlx5_core_dev *mdev = priv->mdev;
3884 
3885 	if (vlan_proto != htons(ETH_P_8021Q))
3886 		return -EPROTONOSUPPORT;
3887 
3888 	return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1,
3889 					   vlan, qos);
3890 }
3891 
mlx5e_set_vf_spoofchk(struct net_device * dev,int vf,bool setting)3892 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting)
3893 {
3894 	struct mlx5e_priv *priv = netdev_priv(dev);
3895 	struct mlx5_core_dev *mdev = priv->mdev;
3896 
3897 	return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting);
3898 }
3899 
mlx5e_set_vf_trust(struct net_device * dev,int vf,bool setting)3900 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting)
3901 {
3902 	struct mlx5e_priv *priv = netdev_priv(dev);
3903 	struct mlx5_core_dev *mdev = priv->mdev;
3904 
3905 	return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting);
3906 }
3907 
mlx5e_set_vf_rate(struct net_device * dev,int vf,int min_tx_rate,int max_tx_rate)3908 static int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate,
3909 			     int max_tx_rate)
3910 {
3911 	struct mlx5e_priv *priv = netdev_priv(dev);
3912 	struct mlx5_core_dev *mdev = priv->mdev;
3913 
3914 	return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1,
3915 					   max_tx_rate, min_tx_rate);
3916 }
3917 
mlx5_vport_link2ifla(u8 esw_link)3918 static int mlx5_vport_link2ifla(u8 esw_link)
3919 {
3920 	switch (esw_link) {
3921 	case MLX5_VPORT_ADMIN_STATE_DOWN:
3922 		return IFLA_VF_LINK_STATE_DISABLE;
3923 	case MLX5_VPORT_ADMIN_STATE_UP:
3924 		return IFLA_VF_LINK_STATE_ENABLE;
3925 	}
3926 	return IFLA_VF_LINK_STATE_AUTO;
3927 }
3928 
mlx5_ifla_link2vport(u8 ifla_link)3929 static int mlx5_ifla_link2vport(u8 ifla_link)
3930 {
3931 	switch (ifla_link) {
3932 	case IFLA_VF_LINK_STATE_DISABLE:
3933 		return MLX5_VPORT_ADMIN_STATE_DOWN;
3934 	case IFLA_VF_LINK_STATE_ENABLE:
3935 		return MLX5_VPORT_ADMIN_STATE_UP;
3936 	}
3937 	return MLX5_VPORT_ADMIN_STATE_AUTO;
3938 }
3939 
mlx5e_set_vf_link_state(struct net_device * dev,int vf,int link_state)3940 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf,
3941 				   int link_state)
3942 {
3943 	struct mlx5e_priv *priv = netdev_priv(dev);
3944 	struct mlx5_core_dev *mdev = priv->mdev;
3945 
3946 	return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1,
3947 					    mlx5_ifla_link2vport(link_state));
3948 }
3949 
mlx5e_get_vf_config(struct net_device * dev,int vf,struct ifla_vf_info * ivi)3950 static int mlx5e_get_vf_config(struct net_device *dev,
3951 			       int vf, struct ifla_vf_info *ivi)
3952 {
3953 	struct mlx5e_priv *priv = netdev_priv(dev);
3954 	struct mlx5_core_dev *mdev = priv->mdev;
3955 	int err;
3956 
3957 	err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi);
3958 	if (err)
3959 		return err;
3960 	ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate);
3961 	return 0;
3962 }
3963 
mlx5e_get_vf_stats(struct net_device * dev,int vf,struct ifla_vf_stats * vf_stats)3964 static int mlx5e_get_vf_stats(struct net_device *dev,
3965 			      int vf, struct ifla_vf_stats *vf_stats)
3966 {
3967 	struct mlx5e_priv *priv = netdev_priv(dev);
3968 	struct mlx5_core_dev *mdev = priv->mdev;
3969 
3970 	return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1,
3971 					    vf_stats);
3972 }
3973 #endif
3974 
3975 struct mlx5e_vxlan_work {
3976 	struct work_struct	work;
3977 	struct mlx5e_priv	*priv;
3978 	u16			port;
3979 };
3980 
mlx5e_vxlan_add_work(struct work_struct * work)3981 static void mlx5e_vxlan_add_work(struct work_struct *work)
3982 {
3983 	struct mlx5e_vxlan_work *vxlan_work =
3984 		container_of(work, struct mlx5e_vxlan_work, work);
3985 	struct mlx5e_priv *priv = vxlan_work->priv;
3986 	u16 port = vxlan_work->port;
3987 
3988 	mutex_lock(&priv->state_lock);
3989 	mlx5_vxlan_add_port(priv->mdev->vxlan, port);
3990 	mutex_unlock(&priv->state_lock);
3991 
3992 	kfree(vxlan_work);
3993 }
3994 
mlx5e_vxlan_del_work(struct work_struct * work)3995 static void mlx5e_vxlan_del_work(struct work_struct *work)
3996 {
3997 	struct mlx5e_vxlan_work *vxlan_work =
3998 		container_of(work, struct mlx5e_vxlan_work, work);
3999 	struct mlx5e_priv *priv         = vxlan_work->priv;
4000 	u16 port = vxlan_work->port;
4001 
4002 	mutex_lock(&priv->state_lock);
4003 	mlx5_vxlan_del_port(priv->mdev->vxlan, port);
4004 	mutex_unlock(&priv->state_lock);
4005 	kfree(vxlan_work);
4006 }
4007 
mlx5e_vxlan_queue_work(struct mlx5e_priv * priv,u16 port,int add)4008 static void mlx5e_vxlan_queue_work(struct mlx5e_priv *priv, u16 port, int add)
4009 {
4010 	struct mlx5e_vxlan_work *vxlan_work;
4011 
4012 	vxlan_work = kmalloc(sizeof(*vxlan_work), GFP_ATOMIC);
4013 	if (!vxlan_work)
4014 		return;
4015 
4016 	if (add)
4017 		INIT_WORK(&vxlan_work->work, mlx5e_vxlan_add_work);
4018 	else
4019 		INIT_WORK(&vxlan_work->work, mlx5e_vxlan_del_work);
4020 
4021 	vxlan_work->priv = priv;
4022 	vxlan_work->port = port;
4023 	queue_work(priv->wq, &vxlan_work->work);
4024 }
4025 
mlx5e_add_vxlan_port(struct net_device * netdev,struct udp_tunnel_info * ti)4026 static void mlx5e_add_vxlan_port(struct net_device *netdev,
4027 				 struct udp_tunnel_info *ti)
4028 {
4029 	struct mlx5e_priv *priv = netdev_priv(netdev);
4030 
4031 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
4032 		return;
4033 
4034 	if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
4035 		return;
4036 
4037 	mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 1);
4038 }
4039 
mlx5e_del_vxlan_port(struct net_device * netdev,struct udp_tunnel_info * ti)4040 static void mlx5e_del_vxlan_port(struct net_device *netdev,
4041 				 struct udp_tunnel_info *ti)
4042 {
4043 	struct mlx5e_priv *priv = netdev_priv(netdev);
4044 
4045 	if (ti->type != UDP_TUNNEL_TYPE_VXLAN)
4046 		return;
4047 
4048 	if (!mlx5_vxlan_allowed(priv->mdev->vxlan))
4049 		return;
4050 
4051 	mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 0);
4052 }
4053 
mlx5e_tunnel_features_check(struct mlx5e_priv * priv,struct sk_buff * skb,netdev_features_t features)4054 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv,
4055 						     struct sk_buff *skb,
4056 						     netdev_features_t features)
4057 {
4058 	unsigned int offset = 0;
4059 	struct udphdr *udph;
4060 	u8 proto;
4061 	u16 port;
4062 
4063 	switch (vlan_get_protocol(skb)) {
4064 	case htons(ETH_P_IP):
4065 		proto = ip_hdr(skb)->protocol;
4066 		break;
4067 	case htons(ETH_P_IPV6):
4068 		proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL);
4069 		break;
4070 	default:
4071 		goto out;
4072 	}
4073 
4074 	switch (proto) {
4075 	case IPPROTO_GRE:
4076 		return features;
4077 	case IPPROTO_UDP:
4078 		udph = udp_hdr(skb);
4079 		port = be16_to_cpu(udph->dest);
4080 
4081 		/* Verify if UDP port is being offloaded by HW */
4082 		if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port))
4083 			return features;
4084 	}
4085 
4086 out:
4087 	/* Disable CSUM and GSO if the udp dport is not offloaded by HW */
4088 	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
4089 }
4090 
mlx5e_features_check(struct sk_buff * skb,struct net_device * netdev,netdev_features_t features)4091 static netdev_features_t mlx5e_features_check(struct sk_buff *skb,
4092 					      struct net_device *netdev,
4093 					      netdev_features_t features)
4094 {
4095 	struct mlx5e_priv *priv = netdev_priv(netdev);
4096 
4097 	features = vlan_features_check(skb, features);
4098 	features = vxlan_features_check(skb, features);
4099 
4100 #ifdef CONFIG_MLX5_EN_IPSEC
4101 	if (mlx5e_ipsec_feature_check(skb, netdev, features))
4102 		return features;
4103 #endif
4104 
4105 	/* Validate if the tunneled packet is being offloaded by HW */
4106 	if (skb->encapsulation &&
4107 	    (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK))
4108 		return mlx5e_tunnel_features_check(priv, skb, features);
4109 
4110 	return features;
4111 }
4112 
mlx5e_tx_timeout_eq_recover(struct net_device * dev,struct mlx5e_txqsq * sq)4113 static bool mlx5e_tx_timeout_eq_recover(struct net_device *dev,
4114 					struct mlx5e_txqsq *sq)
4115 {
4116 	struct mlx5_eq *eq = sq->cq.mcq.eq;
4117 	u32 eqe_count;
4118 
4119 	netdev_err(dev, "EQ 0x%x: Cons = 0x%x, irqn = 0x%x\n",
4120 		   eq->eqn, eq->cons_index, eq->irqn);
4121 
4122 	eqe_count = mlx5_eq_poll_irq_disabled(eq);
4123 	if (!eqe_count)
4124 		return false;
4125 
4126 	netdev_err(dev, "Recover %d eqes on EQ 0x%x\n", eqe_count, eq->eqn);
4127 	sq->channel->stats->eq_rearm++;
4128 	return true;
4129 }
4130 
mlx5e_tx_timeout_work(struct work_struct * work)4131 static void mlx5e_tx_timeout_work(struct work_struct *work)
4132 {
4133 	struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv,
4134 					       tx_timeout_work);
4135 	struct net_device *dev = priv->netdev;
4136 	bool reopen_channels = false;
4137 	int i, err;
4138 
4139 	rtnl_lock();
4140 	mutex_lock(&priv->state_lock);
4141 
4142 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state))
4143 		goto unlock;
4144 
4145 	for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) {
4146 		struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, i);
4147 		struct mlx5e_txqsq *sq = priv->txq2sq[i];
4148 
4149 		if (!netif_xmit_stopped(dev_queue))
4150 			continue;
4151 
4152 		netdev_err(dev,
4153 			   "TX timeout on queue: %d, SQ: 0x%x, CQ: 0x%x, SQ Cons: 0x%x SQ Prod: 0x%x, usecs since last trans: %u\n",
4154 			   i, sq->sqn, sq->cq.mcq.cqn, sq->cc, sq->pc,
4155 			   jiffies_to_usecs(jiffies - dev_queue->trans_start));
4156 
4157 		/* If we recover a lost interrupt, most likely TX timeout will
4158 		 * be resolved, skip reopening channels
4159 		 */
4160 		if (!mlx5e_tx_timeout_eq_recover(dev, sq)) {
4161 			clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state);
4162 			reopen_channels = true;
4163 		}
4164 	}
4165 
4166 	if (!reopen_channels)
4167 		goto unlock;
4168 
4169 	mlx5e_close_locked(dev);
4170 	err = mlx5e_open_locked(dev);
4171 	if (err)
4172 		netdev_err(priv->netdev,
4173 			   "mlx5e_open_locked failed recovering from a tx_timeout, err(%d).\n",
4174 			   err);
4175 
4176 unlock:
4177 	mutex_unlock(&priv->state_lock);
4178 	rtnl_unlock();
4179 }
4180 
mlx5e_tx_timeout(struct net_device * dev)4181 static void mlx5e_tx_timeout(struct net_device *dev)
4182 {
4183 	struct mlx5e_priv *priv = netdev_priv(dev);
4184 
4185 	netdev_err(dev, "TX timeout detected\n");
4186 	queue_work(priv->wq, &priv->tx_timeout_work);
4187 }
4188 
mlx5e_xdp_allowed(struct mlx5e_priv * priv,struct bpf_prog * prog)4189 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog)
4190 {
4191 	struct net_device *netdev = priv->netdev;
4192 	struct mlx5e_channels new_channels = {};
4193 
4194 	if (priv->channels.params.lro_en) {
4195 		netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n");
4196 		return -EINVAL;
4197 	}
4198 
4199 	if (MLX5_IPSEC_DEV(priv->mdev)) {
4200 		netdev_warn(netdev, "can't set XDP with IPSec offload\n");
4201 		return -EINVAL;
4202 	}
4203 
4204 	new_channels.params = priv->channels.params;
4205 	new_channels.params.xdp_prog = prog;
4206 
4207 	if (!mlx5e_rx_is_linear_skb(priv->mdev, &new_channels.params)) {
4208 		netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n",
4209 			    new_channels.params.sw_mtu, MLX5E_XDP_MAX_MTU);
4210 		return -EINVAL;
4211 	}
4212 
4213 	return 0;
4214 }
4215 
mlx5e_xdp_set(struct net_device * netdev,struct bpf_prog * prog)4216 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog)
4217 {
4218 	struct mlx5e_priv *priv = netdev_priv(netdev);
4219 	struct bpf_prog *old_prog;
4220 	bool reset, was_opened;
4221 	int err = 0;
4222 	int i;
4223 
4224 	mutex_lock(&priv->state_lock);
4225 
4226 	if (prog) {
4227 		err = mlx5e_xdp_allowed(priv, prog);
4228 		if (err)
4229 			goto unlock;
4230 	}
4231 
4232 	was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state);
4233 	/* no need for full reset when exchanging programs */
4234 	reset = (!priv->channels.params.xdp_prog || !prog);
4235 
4236 	if (was_opened && reset)
4237 		mlx5e_close_locked(netdev);
4238 	if (was_opened && !reset) {
4239 		/* num_channels is invariant here, so we can take the
4240 		 * batched reference right upfront.
4241 		 */
4242 		prog = bpf_prog_add(prog, priv->channels.num);
4243 		if (IS_ERR(prog)) {
4244 			err = PTR_ERR(prog);
4245 			goto unlock;
4246 		}
4247 	}
4248 
4249 	/* exchange programs, extra prog reference we got from caller
4250 	 * as long as we don't fail from this point onwards.
4251 	 */
4252 	old_prog = xchg(&priv->channels.params.xdp_prog, prog);
4253 	if (old_prog)
4254 		bpf_prog_put(old_prog);
4255 
4256 	if (reset) /* change RQ type according to priv->xdp_prog */
4257 		mlx5e_set_rq_type(priv->mdev, &priv->channels.params);
4258 
4259 	if (was_opened && reset)
4260 		mlx5e_open_locked(netdev);
4261 
4262 	if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset)
4263 		goto unlock;
4264 
4265 	/* exchanging programs w/o reset, we update ref counts on behalf
4266 	 * of the channels RQs here.
4267 	 */
4268 	for (i = 0; i < priv->channels.num; i++) {
4269 		struct mlx5e_channel *c = priv->channels.c[i];
4270 
4271 		clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
4272 		napi_synchronize(&c->napi);
4273 		/* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */
4274 
4275 		old_prog = xchg(&c->rq.xdp_prog, prog);
4276 
4277 		set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state);
4278 		/* napi_schedule in case we have missed anything */
4279 		napi_schedule(&c->napi);
4280 
4281 		if (old_prog)
4282 			bpf_prog_put(old_prog);
4283 	}
4284 
4285 unlock:
4286 	mutex_unlock(&priv->state_lock);
4287 	return err;
4288 }
4289 
mlx5e_xdp_query(struct net_device * dev)4290 static u32 mlx5e_xdp_query(struct net_device *dev)
4291 {
4292 	struct mlx5e_priv *priv = netdev_priv(dev);
4293 	const struct bpf_prog *xdp_prog;
4294 	u32 prog_id = 0;
4295 
4296 	mutex_lock(&priv->state_lock);
4297 	xdp_prog = priv->channels.params.xdp_prog;
4298 	if (xdp_prog)
4299 		prog_id = xdp_prog->aux->id;
4300 	mutex_unlock(&priv->state_lock);
4301 
4302 	return prog_id;
4303 }
4304 
mlx5e_xdp(struct net_device * dev,struct netdev_bpf * xdp)4305 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp)
4306 {
4307 	switch (xdp->command) {
4308 	case XDP_SETUP_PROG:
4309 		return mlx5e_xdp_set(dev, xdp->prog);
4310 	case XDP_QUERY_PROG:
4311 		xdp->prog_id = mlx5e_xdp_query(dev);
4312 		return 0;
4313 	default:
4314 		return -EINVAL;
4315 	}
4316 }
4317 
4318 const struct net_device_ops mlx5e_netdev_ops = {
4319 	.ndo_open                = mlx5e_open,
4320 	.ndo_stop                = mlx5e_close,
4321 	.ndo_start_xmit          = mlx5e_xmit,
4322 	.ndo_setup_tc            = mlx5e_setup_tc,
4323 	.ndo_select_queue        = mlx5e_select_queue,
4324 	.ndo_get_stats64         = mlx5e_get_stats,
4325 	.ndo_set_rx_mode         = mlx5e_set_rx_mode,
4326 	.ndo_set_mac_address     = mlx5e_set_mac,
4327 	.ndo_vlan_rx_add_vid     = mlx5e_vlan_rx_add_vid,
4328 	.ndo_vlan_rx_kill_vid    = mlx5e_vlan_rx_kill_vid,
4329 	.ndo_set_features        = mlx5e_set_features,
4330 	.ndo_fix_features        = mlx5e_fix_features,
4331 	.ndo_change_mtu          = mlx5e_change_nic_mtu,
4332 	.ndo_do_ioctl            = mlx5e_ioctl,
4333 	.ndo_set_tx_maxrate      = mlx5e_set_tx_maxrate,
4334 	.ndo_udp_tunnel_add      = mlx5e_add_vxlan_port,
4335 	.ndo_udp_tunnel_del      = mlx5e_del_vxlan_port,
4336 	.ndo_features_check      = mlx5e_features_check,
4337 	.ndo_tx_timeout          = mlx5e_tx_timeout,
4338 	.ndo_bpf		 = mlx5e_xdp,
4339 	.ndo_xdp_xmit            = mlx5e_xdp_xmit,
4340 #ifdef CONFIG_MLX5_EN_ARFS
4341 	.ndo_rx_flow_steer	 = mlx5e_rx_flow_steer,
4342 #endif
4343 #ifdef CONFIG_MLX5_ESWITCH
4344 	/* SRIOV E-Switch NDOs */
4345 	.ndo_set_vf_mac          = mlx5e_set_vf_mac,
4346 	.ndo_set_vf_vlan         = mlx5e_set_vf_vlan,
4347 	.ndo_set_vf_spoofchk     = mlx5e_set_vf_spoofchk,
4348 	.ndo_set_vf_trust        = mlx5e_set_vf_trust,
4349 	.ndo_set_vf_rate         = mlx5e_set_vf_rate,
4350 	.ndo_get_vf_config       = mlx5e_get_vf_config,
4351 	.ndo_set_vf_link_state   = mlx5e_set_vf_link_state,
4352 	.ndo_get_vf_stats        = mlx5e_get_vf_stats,
4353 	.ndo_has_offload_stats	 = mlx5e_has_offload_stats,
4354 	.ndo_get_offload_stats	 = mlx5e_get_offload_stats,
4355 #endif
4356 };
4357 
mlx5e_check_required_hca_cap(struct mlx5_core_dev * mdev)4358 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev)
4359 {
4360 	if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH)
4361 		return -EOPNOTSUPP;
4362 	if (!MLX5_CAP_GEN(mdev, eth_net_offloads) ||
4363 	    !MLX5_CAP_GEN(mdev, nic_flow_table) ||
4364 	    !MLX5_CAP_ETH(mdev, csum_cap) ||
4365 	    !MLX5_CAP_ETH(mdev, max_lso_cap) ||
4366 	    !MLX5_CAP_ETH(mdev, vlan_cap) ||
4367 	    !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) ||
4368 	    MLX5_CAP_FLOWTABLE(mdev,
4369 			       flow_table_properties_nic_receive.max_ft_level)
4370 			       < 3) {
4371 		mlx5_core_warn(mdev,
4372 			       "Not creating net device, some required device capabilities are missing\n");
4373 		return -EOPNOTSUPP;
4374 	}
4375 	if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable))
4376 		mlx5_core_warn(mdev, "Self loop back prevention is not supported\n");
4377 	if (!MLX5_CAP_GEN(mdev, cq_moderation))
4378 		mlx5_core_warn(mdev, "CQ moderation is not supported\n");
4379 
4380 	return 0;
4381 }
4382 
mlx5e_build_default_indir_rqt(u32 * indirection_rqt,int len,int num_channels)4383 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len,
4384 				   int num_channels)
4385 {
4386 	int i;
4387 
4388 	for (i = 0; i < len; i++)
4389 		indirection_rqt[i] = i % num_channels;
4390 }
4391 
slow_pci_heuristic(struct mlx5_core_dev * mdev)4392 static bool slow_pci_heuristic(struct mlx5_core_dev *mdev)
4393 {
4394 	u32 link_speed = 0;
4395 	u32 pci_bw = 0;
4396 
4397 	mlx5e_port_max_linkspeed(mdev, &link_speed);
4398 	pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL);
4399 	mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n",
4400 			   link_speed, pci_bw);
4401 
4402 #define MLX5E_SLOW_PCI_RATIO (2)
4403 
4404 	return link_speed && pci_bw &&
4405 		link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw;
4406 }
4407 
mlx5e_get_def_tx_moderation(u8 cq_period_mode)4408 static struct net_dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode)
4409 {
4410 	struct net_dim_cq_moder moder;
4411 
4412 	moder.cq_period_mode = cq_period_mode;
4413 	moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS;
4414 	moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC;
4415 	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
4416 		moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE;
4417 
4418 	return moder;
4419 }
4420 
mlx5e_get_def_rx_moderation(u8 cq_period_mode)4421 static struct net_dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode)
4422 {
4423 	struct net_dim_cq_moder moder;
4424 
4425 	moder.cq_period_mode = cq_period_mode;
4426 	moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS;
4427 	moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC;
4428 	if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE)
4429 		moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE;
4430 
4431 	return moder;
4432 }
4433 
mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)4434 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode)
4435 {
4436 	return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ?
4437 		NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE :
4438 		NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE;
4439 }
4440 
mlx5e_set_tx_cq_mode_params(struct mlx5e_params * params,u8 cq_period_mode)4441 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4442 {
4443 	if (params->tx_dim_enabled) {
4444 		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4445 
4446 		params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode);
4447 	} else {
4448 		params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode);
4449 	}
4450 
4451 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER,
4452 			params->tx_cq_moderation.cq_period_mode ==
4453 				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
4454 }
4455 
mlx5e_set_rx_cq_mode_params(struct mlx5e_params * params,u8 cq_period_mode)4456 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode)
4457 {
4458 	if (params->rx_dim_enabled) {
4459 		u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode);
4460 
4461 		params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode);
4462 	} else {
4463 		params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode);
4464 	}
4465 
4466 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER,
4467 			params->rx_cq_moderation.cq_period_mode ==
4468 				MLX5_CQ_PERIOD_MODE_START_FROM_CQE);
4469 }
4470 
mlx5e_choose_lro_timeout(struct mlx5_core_dev * mdev,u32 wanted_timeout)4471 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout)
4472 {
4473 	int i;
4474 
4475 	/* The supported periods are organized in ascending order */
4476 	for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++)
4477 		if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout)
4478 			break;
4479 
4480 	return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]);
4481 }
4482 
mlx5e_build_nic_params(struct mlx5_core_dev * mdev,struct mlx5e_params * params,u16 max_channels,u16 mtu)4483 void mlx5e_build_nic_params(struct mlx5_core_dev *mdev,
4484 			    struct mlx5e_params *params,
4485 			    u16 max_channels, u16 mtu)
4486 {
4487 	u8 rx_cq_period_mode;
4488 
4489 	params->sw_mtu = mtu;
4490 	params->hard_mtu = MLX5E_ETH_HARD_MTU;
4491 	params->num_channels = max_channels;
4492 	params->num_tc       = 1;
4493 
4494 	/* SQ */
4495 	params->log_sq_size = is_kdump_kernel() ?
4496 		MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE :
4497 		MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE;
4498 
4499 	/* set CQE compression */
4500 	params->rx_cqe_compress_def = false;
4501 	if (MLX5_CAP_GEN(mdev, cqe_compression) &&
4502 	    MLX5_CAP_GEN(mdev, vport_group_manager))
4503 		params->rx_cqe_compress_def = slow_pci_heuristic(mdev);
4504 
4505 	MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def);
4506 
4507 	/* RQ */
4508 	/* Prefer Striding RQ, unless any of the following holds:
4509 	 * - Striding RQ configuration is not possible/supported.
4510 	 * - Slow PCI heuristic.
4511 	 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear.
4512 	 */
4513 	if (!slow_pci_heuristic(mdev) &&
4514 	    mlx5e_striding_rq_possible(mdev, params) &&
4515 	    (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ||
4516 	     !mlx5e_rx_is_linear_skb(mdev, params)))
4517 		MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true);
4518 	mlx5e_set_rq_type(mdev, params);
4519 	mlx5e_init_rq_type_params(mdev, params);
4520 
4521 	/* HW LRO */
4522 
4523 	/* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */
4524 	if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
4525 		if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
4526 			params->lro_en = !slow_pci_heuristic(mdev);
4527 	params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT);
4528 
4529 	/* CQ moderation params */
4530 	rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ?
4531 			MLX5_CQ_PERIOD_MODE_START_FROM_CQE :
4532 			MLX5_CQ_PERIOD_MODE_START_FROM_EQE;
4533 	params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4534 	params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation);
4535 	mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode);
4536 	mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE);
4537 
4538 	/* TX inline */
4539 	params->tx_min_inline_mode = mlx5e_params_calculate_tx_min_inline(mdev);
4540 
4541 	/* RSS */
4542 	params->rss_hfunc = ETH_RSS_HASH_XOR;
4543 	netdev_rss_key_fill(params->toeplitz_hash_key, sizeof(params->toeplitz_hash_key));
4544 	mlx5e_build_default_indir_rqt(params->indirection_rqt,
4545 				      MLX5E_INDIR_RQT_SIZE, max_channels);
4546 }
4547 
mlx5e_build_nic_netdev_priv(struct mlx5_core_dev * mdev,struct net_device * netdev,const struct mlx5e_profile * profile,void * ppriv)4548 static void mlx5e_build_nic_netdev_priv(struct mlx5_core_dev *mdev,
4549 					struct net_device *netdev,
4550 					const struct mlx5e_profile *profile,
4551 					void *ppriv)
4552 {
4553 	struct mlx5e_priv *priv = netdev_priv(netdev);
4554 
4555 	priv->mdev        = mdev;
4556 	priv->netdev      = netdev;
4557 	priv->profile     = profile;
4558 	priv->ppriv       = ppriv;
4559 	priv->msglevel    = MLX5E_MSG_LEVEL;
4560 	priv->max_opened_tc = 1;
4561 
4562 	mlx5e_build_nic_params(mdev, &priv->channels.params,
4563 			       profile->max_nch(mdev), netdev->mtu);
4564 
4565 	mutex_init(&priv->state_lock);
4566 
4567 	INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work);
4568 	INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work);
4569 	INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work);
4570 	INIT_DELAYED_WORK(&priv->update_stats_work, mlx5e_update_stats_work);
4571 
4572 	mlx5e_timestamp_init(priv);
4573 }
4574 
mlx5e_set_netdev_dev_addr(struct net_device * netdev)4575 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev)
4576 {
4577 	struct mlx5e_priv *priv = netdev_priv(netdev);
4578 
4579 	mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr);
4580 	if (is_zero_ether_addr(netdev->dev_addr) &&
4581 	    !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) {
4582 		eth_hw_addr_random(netdev);
4583 		mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr);
4584 	}
4585 }
4586 
4587 #if IS_ENABLED(CONFIG_MLX5_ESWITCH)
4588 static const struct switchdev_ops mlx5e_switchdev_ops = {
4589 	.switchdev_port_attr_get	= mlx5e_attr_get,
4590 };
4591 #endif
4592 
mlx5e_build_nic_netdev(struct net_device * netdev)4593 static void mlx5e_build_nic_netdev(struct net_device *netdev)
4594 {
4595 	struct mlx5e_priv *priv = netdev_priv(netdev);
4596 	struct mlx5_core_dev *mdev = priv->mdev;
4597 	bool fcs_supported;
4598 	bool fcs_enabled;
4599 
4600 	SET_NETDEV_DEV(netdev, &mdev->pdev->dev);
4601 
4602 	netdev->netdev_ops = &mlx5e_netdev_ops;
4603 
4604 #ifdef CONFIG_MLX5_CORE_EN_DCB
4605 	if (MLX5_CAP_GEN(mdev, vport_group_manager) && MLX5_CAP_GEN(mdev, qos))
4606 		netdev->dcbnl_ops = &mlx5e_dcbnl_ops;
4607 #endif
4608 
4609 	netdev->watchdog_timeo    = 15 * HZ;
4610 
4611 	netdev->ethtool_ops	  = &mlx5e_ethtool_ops;
4612 
4613 	netdev->vlan_features    |= NETIF_F_SG;
4614 	netdev->vlan_features    |= NETIF_F_IP_CSUM;
4615 	netdev->vlan_features    |= NETIF_F_IPV6_CSUM;
4616 	netdev->vlan_features    |= NETIF_F_GRO;
4617 	netdev->vlan_features    |= NETIF_F_TSO;
4618 	netdev->vlan_features    |= NETIF_F_TSO6;
4619 	netdev->vlan_features    |= NETIF_F_RXCSUM;
4620 	netdev->vlan_features    |= NETIF_F_RXHASH;
4621 
4622 	netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_TX;
4623 	netdev->hw_enc_features  |= NETIF_F_HW_VLAN_CTAG_RX;
4624 
4625 	if (!!MLX5_CAP_ETH(mdev, lro_cap) &&
4626 	    mlx5e_check_fragmented_striding_rq_cap(mdev))
4627 		netdev->vlan_features    |= NETIF_F_LRO;
4628 
4629 	netdev->hw_features       = netdev->vlan_features;
4630 	netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_TX;
4631 	netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_RX;
4632 	netdev->hw_features      |= NETIF_F_HW_VLAN_CTAG_FILTER;
4633 	netdev->hw_features      |= NETIF_F_HW_VLAN_STAG_TX;
4634 
4635 	if (mlx5_vxlan_allowed(mdev->vxlan) || MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
4636 		netdev->hw_enc_features |= NETIF_F_IP_CSUM;
4637 		netdev->hw_enc_features |= NETIF_F_IPV6_CSUM;
4638 		netdev->hw_enc_features |= NETIF_F_TSO;
4639 		netdev->hw_enc_features |= NETIF_F_TSO6;
4640 		netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL;
4641 	}
4642 
4643 	if (mlx5_vxlan_allowed(mdev->vxlan)) {
4644 		netdev->hw_features     |= NETIF_F_GSO_UDP_TUNNEL |
4645 					   NETIF_F_GSO_UDP_TUNNEL_CSUM;
4646 		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL |
4647 					   NETIF_F_GSO_UDP_TUNNEL_CSUM;
4648 		netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM;
4649 	}
4650 
4651 	if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) {
4652 		netdev->hw_features     |= NETIF_F_GSO_GRE |
4653 					   NETIF_F_GSO_GRE_CSUM;
4654 		netdev->hw_enc_features |= NETIF_F_GSO_GRE |
4655 					   NETIF_F_GSO_GRE_CSUM;
4656 		netdev->gso_partial_features |= NETIF_F_GSO_GRE |
4657 						NETIF_F_GSO_GRE_CSUM;
4658 	}
4659 
4660 	netdev->hw_features	                 |= NETIF_F_GSO_PARTIAL;
4661 	netdev->gso_partial_features             |= NETIF_F_GSO_UDP_L4;
4662 	netdev->hw_features                      |= NETIF_F_GSO_UDP_L4;
4663 	netdev->features                         |= NETIF_F_GSO_UDP_L4;
4664 
4665 	mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled);
4666 
4667 	if (fcs_supported)
4668 		netdev->hw_features |= NETIF_F_RXALL;
4669 
4670 	if (MLX5_CAP_ETH(mdev, scatter_fcs))
4671 		netdev->hw_features |= NETIF_F_RXFCS;
4672 
4673 	netdev->features          = netdev->hw_features;
4674 	if (!priv->channels.params.lro_en)
4675 		netdev->features  &= ~NETIF_F_LRO;
4676 
4677 	if (fcs_enabled)
4678 		netdev->features  &= ~NETIF_F_RXALL;
4679 
4680 	if (!priv->channels.params.scatter_fcs_en)
4681 		netdev->features  &= ~NETIF_F_RXFCS;
4682 
4683 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f)
4684 	if (FT_CAP(flow_modify_en) &&
4685 	    FT_CAP(modify_root) &&
4686 	    FT_CAP(identified_miss_table_mode) &&
4687 	    FT_CAP(flow_table_modify)) {
4688 		netdev->hw_features      |= NETIF_F_HW_TC;
4689 #ifdef CONFIG_MLX5_EN_ARFS
4690 		netdev->hw_features	 |= NETIF_F_NTUPLE;
4691 #endif
4692 	}
4693 
4694 	netdev->features         |= NETIF_F_HIGHDMA;
4695 	netdev->features         |= NETIF_F_HW_VLAN_STAG_FILTER;
4696 
4697 	netdev->priv_flags       |= IFF_UNICAST_FLT;
4698 
4699 	mlx5e_set_netdev_dev_addr(netdev);
4700 
4701 #if IS_ENABLED(CONFIG_MLX5_ESWITCH)
4702 	if (MLX5_ESWITCH_MANAGER(mdev))
4703 		netdev->switchdev_ops = &mlx5e_switchdev_ops;
4704 #endif
4705 
4706 	mlx5e_ipsec_build_netdev(priv);
4707 	mlx5e_tls_build_netdev(priv);
4708 }
4709 
mlx5e_create_q_counters(struct mlx5e_priv * priv)4710 static void mlx5e_create_q_counters(struct mlx5e_priv *priv)
4711 {
4712 	struct mlx5_core_dev *mdev = priv->mdev;
4713 	int err;
4714 
4715 	err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter);
4716 	if (err) {
4717 		mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err);
4718 		priv->q_counter = 0;
4719 	}
4720 
4721 	err = mlx5_core_alloc_q_counter(mdev, &priv->drop_rq_q_counter);
4722 	if (err) {
4723 		mlx5_core_warn(mdev, "alloc drop RQ counter failed, %d\n", err);
4724 		priv->drop_rq_q_counter = 0;
4725 	}
4726 }
4727 
mlx5e_destroy_q_counters(struct mlx5e_priv * priv)4728 static void mlx5e_destroy_q_counters(struct mlx5e_priv *priv)
4729 {
4730 	if (priv->q_counter)
4731 		mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter);
4732 
4733 	if (priv->drop_rq_q_counter)
4734 		mlx5_core_dealloc_q_counter(priv->mdev, priv->drop_rq_q_counter);
4735 }
4736 
mlx5e_nic_init(struct mlx5_core_dev * mdev,struct net_device * netdev,const struct mlx5e_profile * profile,void * ppriv)4737 static void mlx5e_nic_init(struct mlx5_core_dev *mdev,
4738 			   struct net_device *netdev,
4739 			   const struct mlx5e_profile *profile,
4740 			   void *ppriv)
4741 {
4742 	struct mlx5e_priv *priv = netdev_priv(netdev);
4743 	int err;
4744 
4745 	mlx5e_build_nic_netdev_priv(mdev, netdev, profile, ppriv);
4746 	err = mlx5e_ipsec_init(priv);
4747 	if (err)
4748 		mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err);
4749 	err = mlx5e_tls_init(priv);
4750 	if (err)
4751 		mlx5_core_err(mdev, "TLS initialization failed, %d\n", err);
4752 	mlx5e_build_nic_netdev(netdev);
4753 	mlx5e_build_tc2txq_maps(priv);
4754 }
4755 
mlx5e_nic_cleanup(struct mlx5e_priv * priv)4756 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv)
4757 {
4758 	mlx5e_tls_cleanup(priv);
4759 	mlx5e_ipsec_cleanup(priv);
4760 }
4761 
mlx5e_init_nic_rx(struct mlx5e_priv * priv)4762 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv)
4763 {
4764 	struct mlx5_core_dev *mdev = priv->mdev;
4765 	int err;
4766 
4767 	err = mlx5e_create_indirect_rqt(priv);
4768 	if (err)
4769 		return err;
4770 
4771 	err = mlx5e_create_direct_rqts(priv);
4772 	if (err)
4773 		goto err_destroy_indirect_rqts;
4774 
4775 	err = mlx5e_create_indirect_tirs(priv);
4776 	if (err)
4777 		goto err_destroy_direct_rqts;
4778 
4779 	err = mlx5e_create_direct_tirs(priv);
4780 	if (err)
4781 		goto err_destroy_indirect_tirs;
4782 
4783 	err = mlx5e_create_flow_steering(priv);
4784 	if (err) {
4785 		mlx5_core_warn(mdev, "create flow steering failed, %d\n", err);
4786 		goto err_destroy_direct_tirs;
4787 	}
4788 
4789 	err = mlx5e_tc_nic_init(priv);
4790 	if (err)
4791 		goto err_destroy_flow_steering;
4792 
4793 	return 0;
4794 
4795 err_destroy_flow_steering:
4796 	mlx5e_destroy_flow_steering(priv);
4797 err_destroy_direct_tirs:
4798 	mlx5e_destroy_direct_tirs(priv);
4799 err_destroy_indirect_tirs:
4800 	mlx5e_destroy_indirect_tirs(priv);
4801 err_destroy_direct_rqts:
4802 	mlx5e_destroy_direct_rqts(priv);
4803 err_destroy_indirect_rqts:
4804 	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4805 	return err;
4806 }
4807 
mlx5e_cleanup_nic_rx(struct mlx5e_priv * priv)4808 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv)
4809 {
4810 	mlx5e_tc_nic_cleanup(priv);
4811 	mlx5e_destroy_flow_steering(priv);
4812 	mlx5e_destroy_direct_tirs(priv);
4813 	mlx5e_destroy_indirect_tirs(priv);
4814 	mlx5e_destroy_direct_rqts(priv);
4815 	mlx5e_destroy_rqt(priv, &priv->indir_rqt);
4816 }
4817 
mlx5e_init_nic_tx(struct mlx5e_priv * priv)4818 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv)
4819 {
4820 	int err;
4821 
4822 	err = mlx5e_create_tises(priv);
4823 	if (err) {
4824 		mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err);
4825 		return err;
4826 	}
4827 
4828 #ifdef CONFIG_MLX5_CORE_EN_DCB
4829 	mlx5e_dcbnl_initialize(priv);
4830 #endif
4831 	return 0;
4832 }
4833 
mlx5e_nic_enable(struct mlx5e_priv * priv)4834 static void mlx5e_nic_enable(struct mlx5e_priv *priv)
4835 {
4836 	struct net_device *netdev = priv->netdev;
4837 	struct mlx5_core_dev *mdev = priv->mdev;
4838 	u16 max_mtu;
4839 
4840 	mlx5e_init_l2_addr(priv);
4841 
4842 	/* Marking the link as currently not needed by the Driver */
4843 	if (!netif_running(netdev))
4844 		mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN);
4845 
4846 	/* MTU range: 68 - hw-specific max */
4847 	netdev->min_mtu = ETH_MIN_MTU;
4848 	mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1);
4849 	netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu);
4850 	mlx5e_set_dev_port_mtu(priv);
4851 
4852 	mlx5_lag_add(mdev, netdev);
4853 
4854 	mlx5e_enable_async_events(priv);
4855 
4856 	if (MLX5_ESWITCH_MANAGER(priv->mdev))
4857 		mlx5e_register_vport_reps(priv);
4858 
4859 	if (netdev->reg_state != NETREG_REGISTERED)
4860 		return;
4861 #ifdef CONFIG_MLX5_CORE_EN_DCB
4862 	mlx5e_dcbnl_init_app(priv);
4863 #endif
4864 
4865 	queue_work(priv->wq, &priv->set_rx_mode_work);
4866 
4867 	rtnl_lock();
4868 	if (netif_running(netdev))
4869 		mlx5e_open(netdev);
4870 	netif_device_attach(netdev);
4871 	rtnl_unlock();
4872 }
4873 
mlx5e_nic_disable(struct mlx5e_priv * priv)4874 static void mlx5e_nic_disable(struct mlx5e_priv *priv)
4875 {
4876 	struct mlx5_core_dev *mdev = priv->mdev;
4877 
4878 #ifdef CONFIG_MLX5_CORE_EN_DCB
4879 	if (priv->netdev->reg_state == NETREG_REGISTERED)
4880 		mlx5e_dcbnl_delete_app(priv);
4881 #endif
4882 
4883 	rtnl_lock();
4884 	if (netif_running(priv->netdev))
4885 		mlx5e_close(priv->netdev);
4886 	netif_device_detach(priv->netdev);
4887 	rtnl_unlock();
4888 
4889 	queue_work(priv->wq, &priv->set_rx_mode_work);
4890 
4891 	if (MLX5_ESWITCH_MANAGER(priv->mdev))
4892 		mlx5e_unregister_vport_reps(priv);
4893 
4894 	mlx5e_disable_async_events(priv);
4895 	mlx5_lag_remove(mdev);
4896 }
4897 
4898 static const struct mlx5e_profile mlx5e_nic_profile = {
4899 	.init		   = mlx5e_nic_init,
4900 	.cleanup	   = mlx5e_nic_cleanup,
4901 	.init_rx	   = mlx5e_init_nic_rx,
4902 	.cleanup_rx	   = mlx5e_cleanup_nic_rx,
4903 	.init_tx	   = mlx5e_init_nic_tx,
4904 	.cleanup_tx	   = mlx5e_cleanup_nic_tx,
4905 	.enable		   = mlx5e_nic_enable,
4906 	.disable	   = mlx5e_nic_disable,
4907 	.update_stats	   = mlx5e_update_ndo_stats,
4908 	.max_nch	   = mlx5e_get_max_num_channels,
4909 	.update_carrier	   = mlx5e_update_carrier,
4910 	.rx_handlers.handle_rx_cqe       = mlx5e_handle_rx_cqe,
4911 	.rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq,
4912 	.max_tc		   = MLX5E_MAX_NUM_TC,
4913 };
4914 
4915 /* mlx5e generic netdev management API (move to en_common.c) */
4916 
mlx5e_create_netdev(struct mlx5_core_dev * mdev,const struct mlx5e_profile * profile,void * ppriv)4917 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev,
4918 				       const struct mlx5e_profile *profile,
4919 				       void *ppriv)
4920 {
4921 	int nch = profile->max_nch(mdev);
4922 	struct net_device *netdev;
4923 	struct mlx5e_priv *priv;
4924 
4925 	netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv),
4926 				    nch * profile->max_tc,
4927 				    nch);
4928 	if (!netdev) {
4929 		mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n");
4930 		return NULL;
4931 	}
4932 
4933 #ifdef CONFIG_MLX5_EN_ARFS
4934 	netdev->rx_cpu_rmap = mdev->rmap;
4935 #endif
4936 
4937 	profile->init(mdev, netdev, profile, ppriv);
4938 
4939 	netif_carrier_off(netdev);
4940 
4941 	priv = netdev_priv(netdev);
4942 
4943 	priv->wq = create_singlethread_workqueue("mlx5e");
4944 	if (!priv->wq)
4945 		goto err_cleanup_nic;
4946 
4947 	return netdev;
4948 
4949 err_cleanup_nic:
4950 	if (profile->cleanup)
4951 		profile->cleanup(priv);
4952 	free_netdev(netdev);
4953 
4954 	return NULL;
4955 }
4956 
mlx5e_attach_netdev(struct mlx5e_priv * priv)4957 int mlx5e_attach_netdev(struct mlx5e_priv *priv)
4958 {
4959 	struct mlx5_core_dev *mdev = priv->mdev;
4960 	const struct mlx5e_profile *profile;
4961 	int err;
4962 
4963 	profile = priv->profile;
4964 	clear_bit(MLX5E_STATE_DESTROYING, &priv->state);
4965 
4966 	err = profile->init_tx(priv);
4967 	if (err)
4968 		goto out;
4969 
4970 	mlx5e_create_q_counters(priv);
4971 
4972 	err = mlx5e_open_drop_rq(priv, &priv->drop_rq);
4973 	if (err) {
4974 		mlx5_core_err(mdev, "open drop rq failed, %d\n", err);
4975 		goto err_destroy_q_counters;
4976 	}
4977 
4978 	err = profile->init_rx(priv);
4979 	if (err)
4980 		goto err_close_drop_rq;
4981 
4982 	if (profile->enable)
4983 		profile->enable(priv);
4984 
4985 	return 0;
4986 
4987 err_close_drop_rq:
4988 	mlx5e_close_drop_rq(&priv->drop_rq);
4989 
4990 err_destroy_q_counters:
4991 	mlx5e_destroy_q_counters(priv);
4992 	profile->cleanup_tx(priv);
4993 
4994 out:
4995 	return err;
4996 }
4997 
mlx5e_detach_netdev(struct mlx5e_priv * priv)4998 void mlx5e_detach_netdev(struct mlx5e_priv *priv)
4999 {
5000 	const struct mlx5e_profile *profile = priv->profile;
5001 
5002 	set_bit(MLX5E_STATE_DESTROYING, &priv->state);
5003 
5004 	if (profile->disable)
5005 		profile->disable(priv);
5006 	flush_workqueue(priv->wq);
5007 
5008 	profile->cleanup_rx(priv);
5009 	mlx5e_close_drop_rq(&priv->drop_rq);
5010 	mlx5e_destroy_q_counters(priv);
5011 	profile->cleanup_tx(priv);
5012 	cancel_delayed_work_sync(&priv->update_stats_work);
5013 }
5014 
mlx5e_destroy_netdev(struct mlx5e_priv * priv)5015 void mlx5e_destroy_netdev(struct mlx5e_priv *priv)
5016 {
5017 	const struct mlx5e_profile *profile = priv->profile;
5018 	struct net_device *netdev = priv->netdev;
5019 
5020 	destroy_workqueue(priv->wq);
5021 	if (profile->cleanup)
5022 		profile->cleanup(priv);
5023 	free_netdev(netdev);
5024 }
5025 
5026 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying
5027  * hardware contexts and to connect it to the current netdev.
5028  */
mlx5e_attach(struct mlx5_core_dev * mdev,void * vpriv)5029 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv)
5030 {
5031 	struct mlx5e_priv *priv = vpriv;
5032 	struct net_device *netdev = priv->netdev;
5033 	int err;
5034 
5035 	if (netif_device_present(netdev))
5036 		return 0;
5037 
5038 	err = mlx5e_create_mdev_resources(mdev);
5039 	if (err)
5040 		return err;
5041 
5042 	err = mlx5e_attach_netdev(priv);
5043 	if (err) {
5044 		mlx5e_destroy_mdev_resources(mdev);
5045 		return err;
5046 	}
5047 
5048 	return 0;
5049 }
5050 
mlx5e_detach(struct mlx5_core_dev * mdev,void * vpriv)5051 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv)
5052 {
5053 	struct mlx5e_priv *priv = vpriv;
5054 	struct net_device *netdev = priv->netdev;
5055 
5056 	if (!netif_device_present(netdev))
5057 		return;
5058 
5059 	mlx5e_detach_netdev(priv);
5060 	mlx5e_destroy_mdev_resources(mdev);
5061 }
5062 
mlx5e_add(struct mlx5_core_dev * mdev)5063 static void *mlx5e_add(struct mlx5_core_dev *mdev)
5064 {
5065 	struct net_device *netdev;
5066 	void *rpriv = NULL;
5067 	void *priv;
5068 	int err;
5069 
5070 	err = mlx5e_check_required_hca_cap(mdev);
5071 	if (err)
5072 		return NULL;
5073 
5074 #ifdef CONFIG_MLX5_ESWITCH
5075 	if (MLX5_ESWITCH_MANAGER(mdev)) {
5076 		rpriv = mlx5e_alloc_nic_rep_priv(mdev);
5077 		if (!rpriv) {
5078 			mlx5_core_warn(mdev, "Failed to alloc NIC rep priv data\n");
5079 			return NULL;
5080 		}
5081 	}
5082 #endif
5083 
5084 	netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, rpriv);
5085 	if (!netdev) {
5086 		mlx5_core_err(mdev, "mlx5e_create_netdev failed\n");
5087 		goto err_free_rpriv;
5088 	}
5089 
5090 	priv = netdev_priv(netdev);
5091 
5092 	err = mlx5e_attach(mdev, priv);
5093 	if (err) {
5094 		mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err);
5095 		goto err_destroy_netdev;
5096 	}
5097 
5098 	err = register_netdev(netdev);
5099 	if (err) {
5100 		mlx5_core_err(mdev, "register_netdev failed, %d\n", err);
5101 		goto err_detach;
5102 	}
5103 
5104 #ifdef CONFIG_MLX5_CORE_EN_DCB
5105 	mlx5e_dcbnl_init_app(priv);
5106 #endif
5107 	return priv;
5108 
5109 err_detach:
5110 	mlx5e_detach(mdev, priv);
5111 err_destroy_netdev:
5112 	mlx5e_destroy_netdev(priv);
5113 err_free_rpriv:
5114 	kfree(rpriv);
5115 	return NULL;
5116 }
5117 
mlx5e_remove(struct mlx5_core_dev * mdev,void * vpriv)5118 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv)
5119 {
5120 	struct mlx5e_priv *priv = vpriv;
5121 	void *ppriv = priv->ppriv;
5122 
5123 #ifdef CONFIG_MLX5_CORE_EN_DCB
5124 	mlx5e_dcbnl_delete_app(priv);
5125 #endif
5126 	unregister_netdev(priv->netdev);
5127 	mlx5e_detach(mdev, vpriv);
5128 	mlx5e_destroy_netdev(priv);
5129 	kfree(ppriv);
5130 }
5131 
mlx5e_get_netdev(void * vpriv)5132 static void *mlx5e_get_netdev(void *vpriv)
5133 {
5134 	struct mlx5e_priv *priv = vpriv;
5135 
5136 	return priv->netdev;
5137 }
5138 
5139 static struct mlx5_interface mlx5e_interface = {
5140 	.add       = mlx5e_add,
5141 	.remove    = mlx5e_remove,
5142 	.attach    = mlx5e_attach,
5143 	.detach    = mlx5e_detach,
5144 	.event     = mlx5e_async_event,
5145 	.protocol  = MLX5_INTERFACE_PROTOCOL_ETH,
5146 	.get_dev   = mlx5e_get_netdev,
5147 };
5148 
mlx5e_init(void)5149 void mlx5e_init(void)
5150 {
5151 	mlx5e_ipsec_build_inverse_table();
5152 	mlx5e_build_ptys2ethtool_map();
5153 	mlx5_register_interface(&mlx5e_interface);
5154 }
5155 
mlx5e_cleanup(void)5156 void mlx5e_cleanup(void)
5157 {
5158 	mlx5_unregister_interface(&mlx5e_interface);
5159 }
5160