1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */
3
4 #include <linux/netdevice.h>
5 #include "en/fs_tt_redirect.h"
6 #include "fs_core.h"
7
8 enum fs_udp_type {
9 FS_IPV4_UDP,
10 FS_IPV6_UDP,
11 FS_UDP_NUM_TYPES,
12 };
13
14 struct mlx5e_fs_udp {
15 struct mlx5e_flow_table tables[FS_UDP_NUM_TYPES];
16 struct mlx5_flow_handle *default_rules[FS_UDP_NUM_TYPES];
17 int ref_cnt;
18 };
19
20 struct mlx5e_fs_any {
21 struct mlx5e_flow_table table;
22 struct mlx5_flow_handle *default_rule;
23 int ref_cnt;
24 };
25
fs_udp_type2str(enum fs_udp_type i)26 static char *fs_udp_type2str(enum fs_udp_type i)
27 {
28 switch (i) {
29 case FS_IPV4_UDP:
30 return "UDP v4";
31 default: /* FS_IPV6_UDP */
32 return "UDP v6";
33 }
34 }
35
fs_udp2tt(enum fs_udp_type i)36 static enum mlx5_traffic_types fs_udp2tt(enum fs_udp_type i)
37 {
38 switch (i) {
39 case FS_IPV4_UDP:
40 return MLX5_TT_IPV4_UDP;
41 default: /* FS_IPV6_UDP */
42 return MLX5_TT_IPV6_UDP;
43 }
44 }
45
tt2fs_udp(enum mlx5_traffic_types i)46 static enum fs_udp_type tt2fs_udp(enum mlx5_traffic_types i)
47 {
48 switch (i) {
49 case MLX5_TT_IPV4_UDP:
50 return FS_IPV4_UDP;
51 case MLX5_TT_IPV6_UDP:
52 return FS_IPV6_UDP;
53 default:
54 return FS_UDP_NUM_TYPES;
55 }
56 }
57
mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle * rule)58 void mlx5e_fs_tt_redirect_del_rule(struct mlx5_flow_handle *rule)
59 {
60 mlx5_del_flow_rules(rule);
61 }
62
fs_udp_set_dport_flow(struct mlx5_flow_spec * spec,enum fs_udp_type type,u16 udp_dport)63 static void fs_udp_set_dport_flow(struct mlx5_flow_spec *spec, enum fs_udp_type type,
64 u16 udp_dport)
65 {
66 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
67 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol);
68 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP);
69 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version);
70 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version,
71 type == FS_IPV4_UDP ? 4 : 6);
72 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport);
73 MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, udp_dport);
74 }
75
76 struct mlx5_flow_handle *
mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv * priv,enum mlx5_traffic_types ttc_type,u32 tir_num,u16 d_port)77 mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv,
78 enum mlx5_traffic_types ttc_type,
79 u32 tir_num, u16 d_port)
80 {
81 enum fs_udp_type type = tt2fs_udp(ttc_type);
82 struct mlx5_flow_destination dest = {};
83 struct mlx5_flow_table *ft = NULL;
84 MLX5_DECLARE_FLOW_ACT(flow_act);
85 struct mlx5_flow_handle *rule;
86 struct mlx5_flow_spec *spec;
87 struct mlx5e_fs_udp *fs_udp;
88 int err;
89
90 if (type == FS_UDP_NUM_TYPES)
91 return ERR_PTR(-EINVAL);
92
93 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
94 if (!spec)
95 return ERR_PTR(-ENOMEM);
96
97 fs_udp = priv->fs.udp;
98 ft = fs_udp->tables[type].t;
99
100 fs_udp_set_dport_flow(spec, type, d_port);
101 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
102 dest.tir_num = tir_num;
103
104 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
105 kvfree(spec);
106
107 if (IS_ERR(rule)) {
108 err = PTR_ERR(rule);
109 netdev_err(priv->netdev, "%s: add %s rule failed, err %d\n",
110 __func__, fs_udp_type2str(type), err);
111 }
112 return rule;
113 }
114
fs_udp_add_default_rule(struct mlx5e_priv * priv,enum fs_udp_type type)115 static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type type)
116 {
117 struct mlx5e_flow_table *fs_udp_t;
118 struct mlx5_flow_destination dest;
119 MLX5_DECLARE_FLOW_ACT(flow_act);
120 struct mlx5_flow_handle *rule;
121 struct mlx5e_fs_udp *fs_udp;
122 int err;
123
124 fs_udp = priv->fs.udp;
125 fs_udp_t = &fs_udp->tables[type];
126
127 dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_udp2tt(type));
128 rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1);
129 if (IS_ERR(rule)) {
130 err = PTR_ERR(rule);
131 netdev_err(priv->netdev,
132 "%s: add default rule failed, fs type=%d, err %d\n",
133 __func__, type, err);
134 return err;
135 }
136
137 fs_udp->default_rules[type] = rule;
138 return 0;
139 }
140
141 #define MLX5E_FS_UDP_NUM_GROUPS (2)
142 #define MLX5E_FS_UDP_GROUP1_SIZE (BIT(16))
143 #define MLX5E_FS_UDP_GROUP2_SIZE (BIT(0))
144 #define MLX5E_FS_UDP_TABLE_SIZE (MLX5E_FS_UDP_GROUP1_SIZE +\
145 MLX5E_FS_UDP_GROUP2_SIZE)
fs_udp_create_groups(struct mlx5e_flow_table * ft,enum fs_udp_type type)146 static int fs_udp_create_groups(struct mlx5e_flow_table *ft, enum fs_udp_type type)
147 {
148 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
149 void *outer_headers_c;
150 int ix = 0;
151 u32 *in;
152 int err;
153 u8 *mc;
154
155 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
156 in = kvzalloc(inlen, GFP_KERNEL);
157 if (!in || !ft->g) {
158 kfree(ft->g);
159 kvfree(in);
160 return -ENOMEM;
161 }
162
163 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
164 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
165 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol);
166 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_version);
167
168 switch (type) {
169 case FS_IPV4_UDP:
170 case FS_IPV6_UDP:
171 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport);
172 break;
173 default:
174 err = -EINVAL;
175 goto out;
176 }
177 /* Match on udp protocol, Ipv4/6 and dport */
178 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
179 MLX5_SET_CFG(in, start_flow_index, ix);
180 ix += MLX5E_FS_UDP_GROUP1_SIZE;
181 MLX5_SET_CFG(in, end_flow_index, ix - 1);
182 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
183 if (IS_ERR(ft->g[ft->num_groups]))
184 goto err;
185 ft->num_groups++;
186
187 /* Default Flow Group */
188 memset(in, 0, inlen);
189 MLX5_SET_CFG(in, start_flow_index, ix);
190 ix += MLX5E_FS_UDP_GROUP2_SIZE;
191 MLX5_SET_CFG(in, end_flow_index, ix - 1);
192 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
193 if (IS_ERR(ft->g[ft->num_groups]))
194 goto err;
195 ft->num_groups++;
196
197 kvfree(in);
198 return 0;
199
200 err:
201 err = PTR_ERR(ft->g[ft->num_groups]);
202 ft->g[ft->num_groups] = NULL;
203 out:
204 kvfree(in);
205
206 return err;
207 }
208
fs_udp_create_table(struct mlx5e_priv * priv,enum fs_udp_type type)209 static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type)
210 {
211 struct mlx5e_flow_table *ft = &priv->fs.udp->tables[type];
212 struct mlx5_flow_table_attr ft_attr = {};
213 int err;
214
215 ft->num_groups = 0;
216
217 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
218 ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL;
219 ft_attr.prio = MLX5E_NIC_PRIO;
220
221 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
222 if (IS_ERR(ft->t)) {
223 err = PTR_ERR(ft->t);
224 ft->t = NULL;
225 return err;
226 }
227
228 netdev_dbg(priv->netdev, "Created fs %s table id %u level %u\n",
229 fs_udp_type2str(type), ft->t->id, ft->t->level);
230
231 err = fs_udp_create_groups(ft, type);
232 if (err)
233 goto err;
234
235 err = fs_udp_add_default_rule(priv, type);
236 if (err)
237 goto err;
238
239 return 0;
240
241 err:
242 mlx5e_destroy_flow_table(ft);
243 return err;
244 }
245
fs_udp_destroy_table(struct mlx5e_fs_udp * fs_udp,int i)246 static void fs_udp_destroy_table(struct mlx5e_fs_udp *fs_udp, int i)
247 {
248 if (IS_ERR_OR_NULL(fs_udp->tables[i].t))
249 return;
250
251 mlx5_del_flow_rules(fs_udp->default_rules[i]);
252 mlx5e_destroy_flow_table(&fs_udp->tables[i]);
253 fs_udp->tables[i].t = NULL;
254 }
255
fs_udp_disable(struct mlx5e_priv * priv)256 static int fs_udp_disable(struct mlx5e_priv *priv)
257 {
258 int err, i;
259
260 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
261 /* Modify ttc rules destination to point back to the indir TIRs */
262 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_udp2tt(i));
263 if (err) {
264 netdev_err(priv->netdev,
265 "%s: modify ttc[%d] default destination failed, err(%d)\n",
266 __func__, fs_udp2tt(i), err);
267 return err;
268 }
269 }
270
271 return 0;
272 }
273
fs_udp_enable(struct mlx5e_priv * priv)274 static int fs_udp_enable(struct mlx5e_priv *priv)
275 {
276 struct mlx5_flow_destination dest = {};
277 int err, i;
278
279 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
280 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
281 dest.ft = priv->fs.udp->tables[i].t;
282
283 /* Modify ttc rules destination to point on the accel_fs FTs */
284 err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_udp2tt(i), &dest);
285 if (err) {
286 netdev_err(priv->netdev,
287 "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
288 __func__, fs_udp2tt(i), err);
289 return err;
290 }
291 }
292 return 0;
293 }
294
mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv * priv)295 void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv)
296 {
297 struct mlx5e_fs_udp *fs_udp = priv->fs.udp;
298 int i;
299
300 if (!fs_udp)
301 return;
302
303 if (--fs_udp->ref_cnt)
304 return;
305
306 fs_udp_disable(priv);
307
308 for (i = 0; i < FS_UDP_NUM_TYPES; i++)
309 fs_udp_destroy_table(fs_udp, i);
310
311 kfree(fs_udp);
312 priv->fs.udp = NULL;
313 }
314
mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv * priv)315 int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv)
316 {
317 int i, err;
318
319 if (priv->fs.udp) {
320 priv->fs.udp->ref_cnt++;
321 return 0;
322 }
323
324 priv->fs.udp = kzalloc(sizeof(*priv->fs.udp), GFP_KERNEL);
325 if (!priv->fs.udp)
326 return -ENOMEM;
327
328 for (i = 0; i < FS_UDP_NUM_TYPES; i++) {
329 err = fs_udp_create_table(priv, i);
330 if (err)
331 goto err_destroy_tables;
332 }
333
334 err = fs_udp_enable(priv);
335 if (err)
336 goto err_destroy_tables;
337
338 priv->fs.udp->ref_cnt = 1;
339
340 return 0;
341
342 err_destroy_tables:
343 while (--i >= 0)
344 fs_udp_destroy_table(priv->fs.udp, i);
345
346 kfree(priv->fs.udp);
347 priv->fs.udp = NULL;
348 return err;
349 }
350
fs_any_set_ethertype_flow(struct mlx5_flow_spec * spec,u16 ether_type)351 static void fs_any_set_ethertype_flow(struct mlx5_flow_spec *spec, u16 ether_type)
352 {
353 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS;
354 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype);
355 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ether_type);
356 }
357
358 struct mlx5_flow_handle *
mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv * priv,u32 tir_num,u16 ether_type)359 mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv,
360 u32 tir_num, u16 ether_type)
361 {
362 struct mlx5_flow_destination dest = {};
363 struct mlx5_flow_table *ft = NULL;
364 MLX5_DECLARE_FLOW_ACT(flow_act);
365 struct mlx5_flow_handle *rule;
366 struct mlx5_flow_spec *spec;
367 struct mlx5e_fs_any *fs_any;
368 int err;
369
370 spec = kvzalloc(sizeof(*spec), GFP_KERNEL);
371 if (!spec)
372 return ERR_PTR(-ENOMEM);
373
374 fs_any = priv->fs.any;
375 ft = fs_any->table.t;
376
377 fs_any_set_ethertype_flow(spec, ether_type);
378 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR;
379 dest.tir_num = tir_num;
380
381 rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1);
382 kvfree(spec);
383
384 if (IS_ERR(rule)) {
385 err = PTR_ERR(rule);
386 netdev_err(priv->netdev, "%s: add ANY rule failed, err %d\n",
387 __func__, err);
388 }
389 return rule;
390 }
391
fs_any_add_default_rule(struct mlx5e_priv * priv)392 static int fs_any_add_default_rule(struct mlx5e_priv *priv)
393 {
394 struct mlx5e_flow_table *fs_any_t;
395 struct mlx5_flow_destination dest;
396 MLX5_DECLARE_FLOW_ACT(flow_act);
397 struct mlx5_flow_handle *rule;
398 struct mlx5e_fs_any *fs_any;
399 int err;
400
401 fs_any = priv->fs.any;
402 fs_any_t = &fs_any->table;
403
404 dest = mlx5_ttc_get_default_dest(priv->fs.ttc, MLX5_TT_ANY);
405 rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1);
406 if (IS_ERR(rule)) {
407 err = PTR_ERR(rule);
408 netdev_err(priv->netdev,
409 "%s: add default rule failed, fs type=ANY, err %d\n",
410 __func__, err);
411 return err;
412 }
413
414 fs_any->default_rule = rule;
415 return 0;
416 }
417
418 #define MLX5E_FS_ANY_NUM_GROUPS (2)
419 #define MLX5E_FS_ANY_GROUP1_SIZE (BIT(16))
420 #define MLX5E_FS_ANY_GROUP2_SIZE (BIT(0))
421 #define MLX5E_FS_ANY_TABLE_SIZE (MLX5E_FS_ANY_GROUP1_SIZE +\
422 MLX5E_FS_ANY_GROUP2_SIZE)
423
fs_any_create_groups(struct mlx5e_flow_table * ft)424 static int fs_any_create_groups(struct mlx5e_flow_table *ft)
425 {
426 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in);
427 void *outer_headers_c;
428 int ix = 0;
429 u32 *in;
430 int err;
431 u8 *mc;
432
433 ft->g = kcalloc(MLX5E_FS_UDP_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL);
434 in = kvzalloc(inlen, GFP_KERNEL);
435 if (!in || !ft->g) {
436 kfree(ft->g);
437 kvfree(in);
438 return -ENOMEM;
439 }
440
441 /* Match on ethertype */
442 mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria);
443 outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers);
444 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ethertype);
445 MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS);
446 MLX5_SET_CFG(in, start_flow_index, ix);
447 ix += MLX5E_FS_ANY_GROUP1_SIZE;
448 MLX5_SET_CFG(in, end_flow_index, ix - 1);
449 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
450 if (IS_ERR(ft->g[ft->num_groups]))
451 goto err;
452 ft->num_groups++;
453
454 /* Default Flow Group */
455 memset(in, 0, inlen);
456 MLX5_SET_CFG(in, start_flow_index, ix);
457 ix += MLX5E_FS_ANY_GROUP2_SIZE;
458 MLX5_SET_CFG(in, end_flow_index, ix - 1);
459 ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in);
460 if (IS_ERR(ft->g[ft->num_groups]))
461 goto err;
462 ft->num_groups++;
463
464 kvfree(in);
465 return 0;
466
467 err:
468 err = PTR_ERR(ft->g[ft->num_groups]);
469 ft->g[ft->num_groups] = NULL;
470 kvfree(in);
471
472 return err;
473 }
474
fs_any_create_table(struct mlx5e_priv * priv)475 static int fs_any_create_table(struct mlx5e_priv *priv)
476 {
477 struct mlx5e_flow_table *ft = &priv->fs.any->table;
478 struct mlx5_flow_table_attr ft_attr = {};
479 int err;
480
481 ft->num_groups = 0;
482
483 ft_attr.max_fte = MLX5E_FS_UDP_TABLE_SIZE;
484 ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL;
485 ft_attr.prio = MLX5E_NIC_PRIO;
486
487 ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr);
488 if (IS_ERR(ft->t)) {
489 err = PTR_ERR(ft->t);
490 ft->t = NULL;
491 return err;
492 }
493
494 netdev_dbg(priv->netdev, "Created fs ANY table id %u level %u\n",
495 ft->t->id, ft->t->level);
496
497 err = fs_any_create_groups(ft);
498 if (err)
499 goto err;
500
501 err = fs_any_add_default_rule(priv);
502 if (err)
503 goto err;
504
505 return 0;
506
507 err:
508 mlx5e_destroy_flow_table(ft);
509 return err;
510 }
511
fs_any_disable(struct mlx5e_priv * priv)512 static int fs_any_disable(struct mlx5e_priv *priv)
513 {
514 int err;
515
516 /* Modify ttc rules destination to point back to the indir TIRs */
517 err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, MLX5_TT_ANY);
518 if (err) {
519 netdev_err(priv->netdev,
520 "%s: modify ttc[%d] default destination failed, err(%d)\n",
521 __func__, MLX5_TT_ANY, err);
522 return err;
523 }
524 return 0;
525 }
526
fs_any_enable(struct mlx5e_priv * priv)527 static int fs_any_enable(struct mlx5e_priv *priv)
528 {
529 struct mlx5_flow_destination dest = {};
530 int err;
531
532 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
533 dest.ft = priv->fs.any->table.t;
534
535 /* Modify ttc rules destination to point on the accel_fs FTs */
536 err = mlx5_ttc_fwd_dest(priv->fs.ttc, MLX5_TT_ANY, &dest);
537 if (err) {
538 netdev_err(priv->netdev,
539 "%s: modify ttc[%d] destination to accel failed, err(%d)\n",
540 __func__, MLX5_TT_ANY, err);
541 return err;
542 }
543 return 0;
544 }
545
fs_any_destroy_table(struct mlx5e_fs_any * fs_any)546 static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any)
547 {
548 if (IS_ERR_OR_NULL(fs_any->table.t))
549 return;
550
551 mlx5_del_flow_rules(fs_any->default_rule);
552 mlx5e_destroy_flow_table(&fs_any->table);
553 fs_any->table.t = NULL;
554 }
555
mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv * priv)556 void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv)
557 {
558 struct mlx5e_fs_any *fs_any = priv->fs.any;
559
560 if (!fs_any)
561 return;
562
563 if (--fs_any->ref_cnt)
564 return;
565
566 fs_any_disable(priv);
567
568 fs_any_destroy_table(fs_any);
569
570 kfree(fs_any);
571 priv->fs.any = NULL;
572 }
573
mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv * priv)574 int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv)
575 {
576 int err;
577
578 if (priv->fs.any) {
579 priv->fs.any->ref_cnt++;
580 return 0;
581 }
582
583 priv->fs.any = kzalloc(sizeof(*priv->fs.any), GFP_KERNEL);
584 if (!priv->fs.any)
585 return -ENOMEM;
586
587 err = fs_any_create_table(priv);
588 if (err)
589 return err;
590
591 err = fs_any_enable(priv);
592 if (err)
593 goto err_destroy_table;
594
595 priv->fs.any->ref_cnt = 1;
596
597 return 0;
598
599 err_destroy_table:
600 fs_any_destroy_table(priv->fs.any);
601
602 kfree(priv->fs.any);
603 priv->fs.any = NULL;
604 return err;
605 }
606
