1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /* Copyright (c) 2018 Mellanox Technologies. */
3
4 #include <net/vxlan.h>
5 #include <net/gre.h>
6 #include <net/geneve.h>
7 #include <net/bareudp.h>
8 #include "en/tc_tun.h"
9 #include "en/tc_priv.h"
10 #include "en_tc.h"
11 #include "rep/tc.h"
12 #include "rep/neigh.h"
13 #include "lag.h"
14 #include "lag_mp.h"
15
16 struct mlx5e_tc_tun_route_attr {
17 struct net_device *out_dev;
18 struct net_device *route_dev;
19 union {
20 struct flowi4 fl4;
21 struct flowi6 fl6;
22 } fl;
23 struct neighbour *n;
24 u8 ttl;
25 };
26
27 #define TC_TUN_ROUTE_ATTR_INIT(name) struct mlx5e_tc_tun_route_attr name = {}
28
mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr * attr)29 static void mlx5e_tc_tun_route_attr_cleanup(struct mlx5e_tc_tun_route_attr *attr)
30 {
31 if (attr->n)
32 neigh_release(attr->n);
33 if (attr->route_dev)
34 dev_put(attr->route_dev);
35 }
36
mlx5e_get_tc_tun(struct net_device * tunnel_dev)37 struct mlx5e_tc_tunnel *mlx5e_get_tc_tun(struct net_device *tunnel_dev)
38 {
39 if (netif_is_vxlan(tunnel_dev))
40 return &vxlan_tunnel;
41 else if (netif_is_geneve(tunnel_dev))
42 return &geneve_tunnel;
43 else if (netif_is_gretap(tunnel_dev) ||
44 netif_is_ip6gretap(tunnel_dev))
45 return &gre_tunnel;
46 else if (netif_is_bareudp(tunnel_dev))
47 return &mplsoudp_tunnel;
48 else
49 return NULL;
50 }
51
get_route_and_out_devs(struct mlx5e_priv * priv,struct net_device * dev,struct net_device ** route_dev,struct net_device ** out_dev)52 static int get_route_and_out_devs(struct mlx5e_priv *priv,
53 struct net_device *dev,
54 struct net_device **route_dev,
55 struct net_device **out_dev)
56 {
57 struct net_device *uplink_dev, *uplink_upper, *real_dev;
58 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
59 bool dst_is_lag_dev;
60
61 real_dev = is_vlan_dev(dev) ? vlan_dev_real_dev(dev) : dev;
62 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
63
64 rcu_read_lock();
65 uplink_upper = netdev_master_upper_dev_get_rcu(uplink_dev);
66 /* mlx5_lag_is_sriov() is a blocking function which can't be called
67 * while holding rcu read lock. Take the net_device for correctness
68 * sake.
69 */
70 if (uplink_upper)
71 dev_hold(uplink_upper);
72 rcu_read_unlock();
73
74 dst_is_lag_dev = (uplink_upper &&
75 netif_is_lag_master(uplink_upper) &&
76 real_dev == uplink_upper &&
77 mlx5_lag_is_sriov(priv->mdev));
78 if (uplink_upper)
79 dev_put(uplink_upper);
80
81 /* if the egress device isn't on the same HW e-switch or
82 * it's a LAG device, use the uplink
83 */
84 *route_dev = dev;
85 if (!netdev_port_same_parent_id(priv->netdev, real_dev) ||
86 dst_is_lag_dev || is_vlan_dev(*route_dev))
87 *out_dev = uplink_dev;
88 else if (mlx5e_eswitch_rep(dev) &&
89 mlx5e_is_valid_eswitch_fwd_dev(priv, dev))
90 *out_dev = *route_dev;
91 else
92 return -EOPNOTSUPP;
93
94 if (!(mlx5e_eswitch_rep(*out_dev) &&
95 mlx5e_is_uplink_rep(netdev_priv(*out_dev))))
96 return -EOPNOTSUPP;
97
98 if (mlx5e_eswitch_uplink_rep(priv->netdev) && *out_dev != priv->netdev)
99 return -EOPNOTSUPP;
100
101 return 0;
102 }
103
mlx5e_route_lookup_ipv4_get(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_tc_tun_route_attr * attr)104 static int mlx5e_route_lookup_ipv4_get(struct mlx5e_priv *priv,
105 struct net_device *mirred_dev,
106 struct mlx5e_tc_tun_route_attr *attr)
107 {
108 struct net_device *route_dev;
109 struct net_device *out_dev;
110 struct neighbour *n;
111 struct rtable *rt;
112
113 #if IS_ENABLED(CONFIG_INET)
114 struct mlx5_core_dev *mdev = priv->mdev;
115 struct net_device *uplink_dev;
116 int ret;
117
118 if (mlx5_lag_is_multipath(mdev)) {
119 struct mlx5_eswitch *esw = mdev->priv.eswitch;
120
121 uplink_dev = mlx5_eswitch_uplink_get_proto_dev(esw, REP_ETH);
122 attr->fl.fl4.flowi4_oif = uplink_dev->ifindex;
123 }
124
125 rt = ip_route_output_key(dev_net(mirred_dev), &attr->fl.fl4);
126 if (IS_ERR(rt))
127 return PTR_ERR(rt);
128
129 if (rt->rt_type != RTN_UNICAST) {
130 ret = -ENETUNREACH;
131 goto err_rt_release;
132 }
133
134 if (mlx5_lag_is_multipath(mdev) && rt->rt_gw_family != AF_INET) {
135 ret = -ENETUNREACH;
136 goto err_rt_release;
137 }
138 #else
139 return -EOPNOTSUPP;
140 #endif
141
142 ret = get_route_and_out_devs(priv, rt->dst.dev, &route_dev, &out_dev);
143 if (ret < 0)
144 goto err_rt_release;
145 dev_hold(route_dev);
146
147 if (!attr->ttl)
148 attr->ttl = ip4_dst_hoplimit(&rt->dst);
149 n = dst_neigh_lookup(&rt->dst, &attr->fl.fl4.daddr);
150 if (!n) {
151 ret = -ENOMEM;
152 goto err_dev_release;
153 }
154
155 ip_rt_put(rt);
156 attr->route_dev = route_dev;
157 attr->out_dev = out_dev;
158 attr->n = n;
159 return 0;
160
161 err_dev_release:
162 dev_put(route_dev);
163 err_rt_release:
164 ip_rt_put(rt);
165 return ret;
166 }
167
mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr * attr)168 static void mlx5e_route_lookup_ipv4_put(struct mlx5e_tc_tun_route_attr *attr)
169 {
170 mlx5e_tc_tun_route_attr_cleanup(attr);
171 }
172
mlx5e_netdev_kind(struct net_device * dev)173 static const char *mlx5e_netdev_kind(struct net_device *dev)
174 {
175 if (dev->rtnl_link_ops)
176 return dev->rtnl_link_ops->kind;
177 else
178 return "unknown";
179 }
180
mlx5e_gen_ip_tunnel_header(char buf[],__u8 * ip_proto,struct mlx5e_encap_entry * e)181 static int mlx5e_gen_ip_tunnel_header(char buf[], __u8 *ip_proto,
182 struct mlx5e_encap_entry *e)
183 {
184 if (!e->tunnel) {
185 pr_warn("mlx5: Cannot generate tunnel header for this tunnel\n");
186 return -EOPNOTSUPP;
187 }
188
189 return e->tunnel->generate_ip_tun_hdr(buf, ip_proto, e);
190 }
191
gen_eth_tnl_hdr(char * buf,struct net_device * dev,struct mlx5e_encap_entry * e,u16 proto)192 static char *gen_eth_tnl_hdr(char *buf, struct net_device *dev,
193 struct mlx5e_encap_entry *e,
194 u16 proto)
195 {
196 struct ethhdr *eth = (struct ethhdr *)buf;
197 char *ip;
198
199 ether_addr_copy(eth->h_dest, e->h_dest);
200 ether_addr_copy(eth->h_source, dev->dev_addr);
201 if (is_vlan_dev(dev)) {
202 struct vlan_hdr *vlan = (struct vlan_hdr *)
203 ((char *)eth + ETH_HLEN);
204 ip = (char *)vlan + VLAN_HLEN;
205 eth->h_proto = vlan_dev_vlan_proto(dev);
206 vlan->h_vlan_TCI = htons(vlan_dev_vlan_id(dev));
207 vlan->h_vlan_encapsulated_proto = htons(proto);
208 } else {
209 eth->h_proto = htons(proto);
210 ip = (char *)eth + ETH_HLEN;
211 }
212
213 return ip;
214 }
215
mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)216 int mlx5e_tc_tun_create_header_ipv4(struct mlx5e_priv *priv,
217 struct net_device *mirred_dev,
218 struct mlx5e_encap_entry *e)
219 {
220 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
221 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
222 struct mlx5_pkt_reformat_params reformat_params;
223 struct mlx5e_neigh m_neigh = {};
224 TC_TUN_ROUTE_ATTR_INIT(attr);
225 int ipv4_encap_size;
226 char *encap_header;
227 struct iphdr *ip;
228 u8 nud_state;
229 int err;
230
231 /* add the IP fields */
232 attr.fl.fl4.flowi4_tos = tun_key->tos;
233 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
234 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
235 attr.ttl = tun_key->ttl;
236
237 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
238 if (err)
239 return err;
240
241 ipv4_encap_size =
242 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
243 sizeof(struct iphdr) +
244 e->tunnel->calc_hlen(e);
245
246 if (max_encap_size < ipv4_encap_size) {
247 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
248 ipv4_encap_size, max_encap_size);
249 err = -EOPNOTSUPP;
250 goto release_neigh;
251 }
252
253 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
254 if (!encap_header) {
255 err = -ENOMEM;
256 goto release_neigh;
257 }
258
259 m_neigh.family = attr.n->ops->family;
260 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
261 e->out_dev = attr.out_dev;
262 e->route_dev_ifindex = attr.route_dev->ifindex;
263
264 /* It's important to add the neigh to the hash table before checking
265 * the neigh validity state. So if we'll get a notification, in case the
266 * neigh changes it's validity state, we would find the relevant neigh
267 * in the hash.
268 */
269 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
270 if (err)
271 goto free_encap;
272
273 read_lock_bh(&attr.n->lock);
274 nud_state = attr.n->nud_state;
275 ether_addr_copy(e->h_dest, attr.n->ha);
276 read_unlock_bh(&attr.n->lock);
277
278 /* add ethernet header */
279 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
280 ETH_P_IP);
281
282 /* add ip header */
283 ip->tos = tun_key->tos;
284 ip->version = 0x4;
285 ip->ihl = 0x5;
286 ip->ttl = attr.ttl;
287 ip->daddr = attr.fl.fl4.daddr;
288 ip->saddr = attr.fl.fl4.saddr;
289
290 /* add tunneling protocol header */
291 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
292 &ip->protocol, e);
293 if (err)
294 goto destroy_neigh_entry;
295
296 e->encap_size = ipv4_encap_size;
297 e->encap_header = encap_header;
298
299 if (!(nud_state & NUD_VALID)) {
300 neigh_event_send(attr.n, NULL);
301 /* the encap entry will be made valid on neigh update event
302 * and not used before that.
303 */
304 goto release_neigh;
305 }
306
307 memset(&reformat_params, 0, sizeof(reformat_params));
308 reformat_params.type = e->reformat_type;
309 reformat_params.size = ipv4_encap_size;
310 reformat_params.data = encap_header;
311 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
312 MLX5_FLOW_NAMESPACE_FDB);
313 if (IS_ERR(e->pkt_reformat)) {
314 err = PTR_ERR(e->pkt_reformat);
315 goto destroy_neigh_entry;
316 }
317
318 e->flags |= MLX5_ENCAP_ENTRY_VALID;
319 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
320 mlx5e_route_lookup_ipv4_put(&attr);
321 return err;
322
323 destroy_neigh_entry:
324 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
325 free_encap:
326 kfree(encap_header);
327 release_neigh:
328 mlx5e_route_lookup_ipv4_put(&attr);
329 return err;
330 }
331
mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)332 int mlx5e_tc_tun_update_header_ipv4(struct mlx5e_priv *priv,
333 struct net_device *mirred_dev,
334 struct mlx5e_encap_entry *e)
335 {
336 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
337 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
338 struct mlx5_pkt_reformat_params reformat_params;
339 TC_TUN_ROUTE_ATTR_INIT(attr);
340 int ipv4_encap_size;
341 char *encap_header;
342 struct iphdr *ip;
343 u8 nud_state;
344 int err;
345
346 /* add the IP fields */
347 attr.fl.fl4.flowi4_tos = tun_key->tos;
348 attr.fl.fl4.daddr = tun_key->u.ipv4.dst;
349 attr.fl.fl4.saddr = tun_key->u.ipv4.src;
350 attr.ttl = tun_key->ttl;
351
352 err = mlx5e_route_lookup_ipv4_get(priv, mirred_dev, &attr);
353 if (err)
354 return err;
355
356 ipv4_encap_size =
357 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
358 sizeof(struct iphdr) +
359 e->tunnel->calc_hlen(e);
360
361 if (max_encap_size < ipv4_encap_size) {
362 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
363 ipv4_encap_size, max_encap_size);
364 err = -EOPNOTSUPP;
365 goto release_neigh;
366 }
367
368 encap_header = kzalloc(ipv4_encap_size, GFP_KERNEL);
369 if (!encap_header) {
370 err = -ENOMEM;
371 goto release_neigh;
372 }
373
374 e->route_dev_ifindex = attr.route_dev->ifindex;
375
376 read_lock_bh(&attr.n->lock);
377 nud_state = attr.n->nud_state;
378 ether_addr_copy(e->h_dest, attr.n->ha);
379 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
380 read_unlock_bh(&attr.n->lock);
381
382 /* add ethernet header */
383 ip = (struct iphdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
384 ETH_P_IP);
385
386 /* add ip header */
387 ip->tos = tun_key->tos;
388 ip->version = 0x4;
389 ip->ihl = 0x5;
390 ip->ttl = attr.ttl;
391 ip->daddr = attr.fl.fl4.daddr;
392 ip->saddr = attr.fl.fl4.saddr;
393
394 /* add tunneling protocol header */
395 err = mlx5e_gen_ip_tunnel_header((char *)ip + sizeof(struct iphdr),
396 &ip->protocol, e);
397 if (err)
398 goto free_encap;
399
400 e->encap_size = ipv4_encap_size;
401 kfree(e->encap_header);
402 e->encap_header = encap_header;
403
404 if (!(nud_state & NUD_VALID)) {
405 neigh_event_send(attr.n, NULL);
406 /* the encap entry will be made valid on neigh update event
407 * and not used before that.
408 */
409 goto release_neigh;
410 }
411
412 memset(&reformat_params, 0, sizeof(reformat_params));
413 reformat_params.type = e->reformat_type;
414 reformat_params.size = ipv4_encap_size;
415 reformat_params.data = encap_header;
416 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
417 MLX5_FLOW_NAMESPACE_FDB);
418 if (IS_ERR(e->pkt_reformat)) {
419 err = PTR_ERR(e->pkt_reformat);
420 goto free_encap;
421 }
422
423 e->flags |= MLX5_ENCAP_ENTRY_VALID;
424 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
425 mlx5e_route_lookup_ipv4_put(&attr);
426 return err;
427
428 free_encap:
429 kfree(encap_header);
430 release_neigh:
431 mlx5e_route_lookup_ipv4_put(&attr);
432 return err;
433 }
434
435 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
mlx5e_route_lookup_ipv6_get(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_tc_tun_route_attr * attr)436 static int mlx5e_route_lookup_ipv6_get(struct mlx5e_priv *priv,
437 struct net_device *mirred_dev,
438 struct mlx5e_tc_tun_route_attr *attr)
439 {
440 struct net_device *route_dev;
441 struct net_device *out_dev;
442 struct dst_entry *dst;
443 struct neighbour *n;
444 int ret;
445
446 dst = ipv6_stub->ipv6_dst_lookup_flow(dev_net(mirred_dev), NULL, &attr->fl.fl6,
447 NULL);
448 if (IS_ERR(dst))
449 return PTR_ERR(dst);
450
451 if (!attr->ttl)
452 attr->ttl = ip6_dst_hoplimit(dst);
453
454 ret = get_route_and_out_devs(priv, dst->dev, &route_dev, &out_dev);
455 if (ret < 0)
456 goto err_dst_release;
457
458 dev_hold(route_dev);
459 n = dst_neigh_lookup(dst, &attr->fl.fl6.daddr);
460 if (!n) {
461 ret = -ENOMEM;
462 goto err_dev_release;
463 }
464
465 dst_release(dst);
466 attr->out_dev = out_dev;
467 attr->route_dev = route_dev;
468 attr->n = n;
469 return 0;
470
471 err_dev_release:
472 dev_put(route_dev);
473 err_dst_release:
474 dst_release(dst);
475 return ret;
476 }
477
mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr * attr)478 static void mlx5e_route_lookup_ipv6_put(struct mlx5e_tc_tun_route_attr *attr)
479 {
480 mlx5e_tc_tun_route_attr_cleanup(attr);
481 }
482
mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)483 int mlx5e_tc_tun_create_header_ipv6(struct mlx5e_priv *priv,
484 struct net_device *mirred_dev,
485 struct mlx5e_encap_entry *e)
486 {
487 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
488 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
489 struct mlx5_pkt_reformat_params reformat_params;
490 struct mlx5e_neigh m_neigh = {};
491 TC_TUN_ROUTE_ATTR_INIT(attr);
492 struct ipv6hdr *ip6h;
493 int ipv6_encap_size;
494 char *encap_header;
495 u8 nud_state;
496 int err;
497
498 attr.ttl = tun_key->ttl;
499 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
500 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
501 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
502
503 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
504 if (err)
505 return err;
506
507 ipv6_encap_size =
508 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
509 sizeof(struct ipv6hdr) +
510 e->tunnel->calc_hlen(e);
511
512 if (max_encap_size < ipv6_encap_size) {
513 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
514 ipv6_encap_size, max_encap_size);
515 err = -EOPNOTSUPP;
516 goto release_neigh;
517 }
518
519 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
520 if (!encap_header) {
521 err = -ENOMEM;
522 goto release_neigh;
523 }
524
525 m_neigh.family = attr.n->ops->family;
526 memcpy(&m_neigh.dst_ip, attr.n->primary_key, attr.n->tbl->key_len);
527 e->out_dev = attr.out_dev;
528 e->route_dev_ifindex = attr.route_dev->ifindex;
529
530 /* It's important to add the neigh to the hash table before checking
531 * the neigh validity state. So if we'll get a notification, in case the
532 * neigh changes it's validity state, we would find the relevant neigh
533 * in the hash.
534 */
535 err = mlx5e_rep_encap_entry_attach(netdev_priv(attr.out_dev), e, &m_neigh, attr.n->dev);
536 if (err)
537 goto free_encap;
538
539 read_lock_bh(&attr.n->lock);
540 nud_state = attr.n->nud_state;
541 ether_addr_copy(e->h_dest, attr.n->ha);
542 read_unlock_bh(&attr.n->lock);
543
544 /* add ethernet header */
545 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
546 ETH_P_IPV6);
547
548 /* add ip header */
549 ip6_flow_hdr(ip6h, tun_key->tos, 0);
550 /* the HW fills up ipv6 payload len */
551 ip6h->hop_limit = attr.ttl;
552 ip6h->daddr = attr.fl.fl6.daddr;
553 ip6h->saddr = attr.fl.fl6.saddr;
554
555 /* add tunneling protocol header */
556 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
557 &ip6h->nexthdr, e);
558 if (err)
559 goto destroy_neigh_entry;
560
561 e->encap_size = ipv6_encap_size;
562 e->encap_header = encap_header;
563
564 if (!(nud_state & NUD_VALID)) {
565 neigh_event_send(attr.n, NULL);
566 /* the encap entry will be made valid on neigh update event
567 * and not used before that.
568 */
569 goto release_neigh;
570 }
571
572 memset(&reformat_params, 0, sizeof(reformat_params));
573 reformat_params.type = e->reformat_type;
574 reformat_params.size = ipv6_encap_size;
575 reformat_params.data = encap_header;
576 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
577 MLX5_FLOW_NAMESPACE_FDB);
578 if (IS_ERR(e->pkt_reformat)) {
579 err = PTR_ERR(e->pkt_reformat);
580 goto destroy_neigh_entry;
581 }
582
583 e->flags |= MLX5_ENCAP_ENTRY_VALID;
584 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
585 mlx5e_route_lookup_ipv6_put(&attr);
586 return err;
587
588 destroy_neigh_entry:
589 mlx5e_rep_encap_entry_detach(netdev_priv(e->out_dev), e);
590 free_encap:
591 kfree(encap_header);
592 release_neigh:
593 mlx5e_route_lookup_ipv6_put(&attr);
594 return err;
595 }
596
mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv * priv,struct net_device * mirred_dev,struct mlx5e_encap_entry * e)597 int mlx5e_tc_tun_update_header_ipv6(struct mlx5e_priv *priv,
598 struct net_device *mirred_dev,
599 struct mlx5e_encap_entry *e)
600 {
601 int max_encap_size = MLX5_CAP_ESW(priv->mdev, max_encap_header_size);
602 const struct ip_tunnel_key *tun_key = &e->tun_info->key;
603 struct mlx5_pkt_reformat_params reformat_params;
604 TC_TUN_ROUTE_ATTR_INIT(attr);
605 struct ipv6hdr *ip6h;
606 int ipv6_encap_size;
607 char *encap_header;
608 u8 nud_state;
609 int err;
610
611 attr.ttl = tun_key->ttl;
612
613 attr.fl.fl6.flowlabel = ip6_make_flowinfo(RT_TOS(tun_key->tos), tun_key->label);
614 attr.fl.fl6.daddr = tun_key->u.ipv6.dst;
615 attr.fl.fl6.saddr = tun_key->u.ipv6.src;
616
617 err = mlx5e_route_lookup_ipv6_get(priv, mirred_dev, &attr);
618 if (err)
619 return err;
620
621 ipv6_encap_size =
622 (is_vlan_dev(attr.route_dev) ? VLAN_ETH_HLEN : ETH_HLEN) +
623 sizeof(struct ipv6hdr) +
624 e->tunnel->calc_hlen(e);
625
626 if (max_encap_size < ipv6_encap_size) {
627 mlx5_core_warn(priv->mdev, "encap size %d too big, max supported is %d\n",
628 ipv6_encap_size, max_encap_size);
629 err = -EOPNOTSUPP;
630 goto release_neigh;
631 }
632
633 encap_header = kzalloc(ipv6_encap_size, GFP_KERNEL);
634 if (!encap_header) {
635 err = -ENOMEM;
636 goto release_neigh;
637 }
638
639 e->route_dev_ifindex = attr.route_dev->ifindex;
640
641 read_lock_bh(&attr.n->lock);
642 nud_state = attr.n->nud_state;
643 ether_addr_copy(e->h_dest, attr.n->ha);
644 WRITE_ONCE(e->nhe->neigh_dev, attr.n->dev);
645 read_unlock_bh(&attr.n->lock);
646
647 /* add ethernet header */
648 ip6h = (struct ipv6hdr *)gen_eth_tnl_hdr(encap_header, attr.route_dev, e,
649 ETH_P_IPV6);
650
651 /* add ip header */
652 ip6_flow_hdr(ip6h, tun_key->tos, 0);
653 /* the HW fills up ipv6 payload len */
654 ip6h->hop_limit = attr.ttl;
655 ip6h->daddr = attr.fl.fl6.daddr;
656 ip6h->saddr = attr.fl.fl6.saddr;
657
658 /* add tunneling protocol header */
659 err = mlx5e_gen_ip_tunnel_header((char *)ip6h + sizeof(struct ipv6hdr),
660 &ip6h->nexthdr, e);
661 if (err)
662 goto free_encap;
663
664 e->encap_size = ipv6_encap_size;
665 kfree(e->encap_header);
666 e->encap_header = encap_header;
667
668 if (!(nud_state & NUD_VALID)) {
669 neigh_event_send(attr.n, NULL);
670 /* the encap entry will be made valid on neigh update event
671 * and not used before that.
672 */
673 goto release_neigh;
674 }
675
676 memset(&reformat_params, 0, sizeof(reformat_params));
677 reformat_params.type = e->reformat_type;
678 reformat_params.size = ipv6_encap_size;
679 reformat_params.data = encap_header;
680 e->pkt_reformat = mlx5_packet_reformat_alloc(priv->mdev, &reformat_params,
681 MLX5_FLOW_NAMESPACE_FDB);
682 if (IS_ERR(e->pkt_reformat)) {
683 err = PTR_ERR(e->pkt_reformat);
684 goto free_encap;
685 }
686
687 e->flags |= MLX5_ENCAP_ENTRY_VALID;
688 mlx5e_rep_queue_neigh_stats_work(netdev_priv(attr.out_dev));
689 mlx5e_route_lookup_ipv6_put(&attr);
690 return err;
691
692 free_encap:
693 kfree(encap_header);
694 release_neigh:
695 mlx5e_route_lookup_ipv6_put(&attr);
696 return err;
697 }
698 #endif
699
mlx5e_tc_tun_route_lookup(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct mlx5_flow_attr * flow_attr)700 int mlx5e_tc_tun_route_lookup(struct mlx5e_priv *priv,
701 struct mlx5_flow_spec *spec,
702 struct mlx5_flow_attr *flow_attr)
703 {
704 struct mlx5_esw_flow_attr *esw_attr = flow_attr->esw_attr;
705 TC_TUN_ROUTE_ATTR_INIT(attr);
706 u16 vport_num;
707 int err = 0;
708
709 if (flow_attr->tun_ip_version == 4) {
710 /* Addresses are swapped for decap */
711 attr.fl.fl4.saddr = esw_attr->rx_tun_attr->dst_ip.v4;
712 attr.fl.fl4.daddr = esw_attr->rx_tun_attr->src_ip.v4;
713 err = mlx5e_route_lookup_ipv4_get(priv, priv->netdev, &attr);
714 }
715 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
716 else if (flow_attr->tun_ip_version == 6) {
717 /* Addresses are swapped for decap */
718 attr.fl.fl6.saddr = esw_attr->rx_tun_attr->dst_ip.v6;
719 attr.fl.fl6.daddr = esw_attr->rx_tun_attr->src_ip.v6;
720 err = mlx5e_route_lookup_ipv6_get(priv, priv->netdev, &attr);
721 }
722 #endif
723 else
724 return 0;
725
726 if (err)
727 return err;
728
729 if (attr.route_dev->netdev_ops != &mlx5e_netdev_ops ||
730 !mlx5e_tc_is_vf_tunnel(attr.out_dev, attr.route_dev))
731 goto out;
732
733 err = mlx5e_tc_query_route_vport(attr.out_dev, attr.route_dev, &vport_num);
734 if (err)
735 goto out;
736
737 esw_attr->rx_tun_attr->vni = MLX5_GET(fte_match_param, spec->match_value,
738 misc_parameters.vxlan_vni);
739 esw_attr->rx_tun_attr->decap_vport = vport_num;
740
741 out:
742 if (flow_attr->tun_ip_version == 4)
743 mlx5e_route_lookup_ipv4_put(&attr);
744 #if IS_ENABLED(CONFIG_INET) && IS_ENABLED(CONFIG_IPV6)
745 else if (flow_attr->tun_ip_version == 6)
746 mlx5e_route_lookup_ipv6_put(&attr);
747 #endif
748 return err;
749 }
750
mlx5e_tc_tun_device_to_offload(struct mlx5e_priv * priv,struct net_device * netdev)751 bool mlx5e_tc_tun_device_to_offload(struct mlx5e_priv *priv,
752 struct net_device *netdev)
753 {
754 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(netdev);
755
756 if (tunnel && tunnel->can_offload(priv))
757 return true;
758 else
759 return false;
760 }
761
mlx5e_tc_tun_init_encap_attr(struct net_device * tunnel_dev,struct mlx5e_priv * priv,struct mlx5e_encap_entry * e,struct netlink_ext_ack * extack)762 int mlx5e_tc_tun_init_encap_attr(struct net_device *tunnel_dev,
763 struct mlx5e_priv *priv,
764 struct mlx5e_encap_entry *e,
765 struct netlink_ext_ack *extack)
766 {
767 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(tunnel_dev);
768
769 if (!tunnel) {
770 e->reformat_type = -1;
771 return -EOPNOTSUPP;
772 }
773
774 return tunnel->init_encap_attr(tunnel_dev, priv, e, extack);
775 }
776
mlx5e_tc_tun_parse(struct net_device * filter_dev,struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,u8 * match_level)777 int mlx5e_tc_tun_parse(struct net_device *filter_dev,
778 struct mlx5e_priv *priv,
779 struct mlx5_flow_spec *spec,
780 struct flow_cls_offload *f,
781 u8 *match_level)
782 {
783 struct mlx5e_tc_tunnel *tunnel = mlx5e_get_tc_tun(filter_dev);
784 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
785 void *headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria,
786 outer_headers);
787 void *headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value,
788 outer_headers);
789 struct netlink_ext_ack *extack = f->common.extack;
790 int err = 0;
791
792 if (!tunnel) {
793 netdev_warn(priv->netdev,
794 "decapsulation offload is not supported for %s net device\n",
795 mlx5e_netdev_kind(filter_dev));
796 err = -EOPNOTSUPP;
797 goto out;
798 }
799
800 *match_level = tunnel->match_level;
801
802 if (tunnel->parse_udp_ports) {
803 err = tunnel->parse_udp_ports(priv, spec, f,
804 headers_c, headers_v);
805 if (err)
806 goto out;
807 }
808
809 if (tunnel->parse_tunnel) {
810 err = tunnel->parse_tunnel(priv, spec, f,
811 headers_c, headers_v);
812 if (err)
813 goto out;
814 }
815
816 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_CONTROL)) {
817 struct flow_dissector_key_basic key_basic = {};
818 struct flow_dissector_key_basic mask_basic = {
819 .n_proto = htons(0xFFFF),
820 };
821 struct flow_match_basic match_basic = {
822 .key = &key_basic, .mask = &mask_basic,
823 };
824 struct flow_match_control match;
825 u16 addr_type;
826
827 flow_rule_match_enc_control(rule, &match);
828 addr_type = match.key->addr_type;
829
830 /* For tunnel addr_type used same key id`s as for non-tunnel */
831 if (addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
832 struct flow_match_ipv4_addrs match;
833
834 flow_rule_match_enc_ipv4_addrs(rule, &match);
835 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
836 src_ipv4_src_ipv6.ipv4_layout.ipv4,
837 ntohl(match.mask->src));
838 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
839 src_ipv4_src_ipv6.ipv4_layout.ipv4,
840 ntohl(match.key->src));
841
842 MLX5_SET(fte_match_set_lyr_2_4, headers_c,
843 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
844 ntohl(match.mask->dst));
845 MLX5_SET(fte_match_set_lyr_2_4, headers_v,
846 dst_ipv4_dst_ipv6.ipv4_layout.ipv4,
847 ntohl(match.key->dst));
848
849 key_basic.n_proto = htons(ETH_P_IP);
850 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
851 headers_c, headers_v);
852 } else if (addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
853 struct flow_match_ipv6_addrs match;
854
855 flow_rule_match_enc_ipv6_addrs(rule, &match);
856 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
857 src_ipv4_src_ipv6.ipv6_layout.ipv6),
858 &match.mask->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
859 ipv6));
860 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
861 src_ipv4_src_ipv6.ipv6_layout.ipv6),
862 &match.key->src, MLX5_FLD_SZ_BYTES(ipv6_layout,
863 ipv6));
864
865 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_c,
866 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
867 &match.mask->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
868 ipv6));
869 memcpy(MLX5_ADDR_OF(fte_match_set_lyr_2_4, headers_v,
870 dst_ipv4_dst_ipv6.ipv6_layout.ipv6),
871 &match.key->dst, MLX5_FLD_SZ_BYTES(ipv6_layout,
872 ipv6));
873
874 key_basic.n_proto = htons(ETH_P_IPV6);
875 mlx5e_tc_set_ethertype(priv->mdev, &match_basic, true,
876 headers_c, headers_v);
877 }
878 }
879
880 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_IP)) {
881 struct flow_match_ip match;
882
883 flow_rule_match_enc_ip(rule, &match);
884 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_ecn,
885 match.mask->tos & 0x3);
886 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_ecn,
887 match.key->tos & 0x3);
888
889 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ip_dscp,
890 match.mask->tos >> 2);
891 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_dscp,
892 match.key->tos >> 2);
893
894 MLX5_SET(fte_match_set_lyr_2_4, headers_c, ttl_hoplimit,
895 match.mask->ttl);
896 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ttl_hoplimit,
897 match.key->ttl);
898
899 if (match.mask->ttl &&
900 !MLX5_CAP_ESW_FLOWTABLE_FDB
901 (priv->mdev,
902 ft_field_support.outer_ipv4_ttl)) {
903 NL_SET_ERR_MSG_MOD(extack,
904 "Matching on TTL is not supported");
905 err = -EOPNOTSUPP;
906 goto out;
907 }
908 }
909
910 /* let software handle IP fragments */
911 MLX5_SET(fte_match_set_lyr_2_4, headers_c, frag, 1);
912 MLX5_SET(fte_match_set_lyr_2_4, headers_v, frag, 0);
913
914 return 0;
915
916 out:
917 return err;
918 }
919
mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv * priv,struct mlx5_flow_spec * spec,struct flow_cls_offload * f,void * headers_c,void * headers_v)920 int mlx5e_tc_tun_parse_udp_ports(struct mlx5e_priv *priv,
921 struct mlx5_flow_spec *spec,
922 struct flow_cls_offload *f,
923 void *headers_c,
924 void *headers_v)
925 {
926 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
927 struct netlink_ext_ack *extack = f->common.extack;
928 struct flow_match_ports enc_ports;
929
930 /* Full udp dst port must be given */
931
932 if (!flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ENC_PORTS)) {
933 NL_SET_ERR_MSG_MOD(extack,
934 "UDP tunnel decap filter must include enc_dst_port condition");
935 netdev_warn(priv->netdev,
936 "UDP tunnel decap filter must include enc_dst_port condition\n");
937 return -EOPNOTSUPP;
938 }
939
940 flow_rule_match_enc_ports(rule, &enc_ports);
941
942 if (memchr_inv(&enc_ports.mask->dst, 0xff,
943 sizeof(enc_ports.mask->dst))) {
944 NL_SET_ERR_MSG_MOD(extack,
945 "UDP tunnel decap filter must match enc_dst_port fully");
946 netdev_warn(priv->netdev,
947 "UDP tunnel decap filter must match enc_dst_port fully\n");
948 return -EOPNOTSUPP;
949 }
950
951 /* match on UDP protocol and dst port number */
952
953 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, headers_c, ip_protocol);
954 MLX5_SET(fte_match_set_lyr_2_4, headers_v, ip_protocol, IPPROTO_UDP);
955
956 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_dport,
957 ntohs(enc_ports.mask->dst));
958 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_dport,
959 ntohs(enc_ports.key->dst));
960
961 /* UDP src port on outer header is generated by HW,
962 * so it is probably a bad idea to request matching it.
963 * Nonetheless, it is allowed.
964 */
965
966 MLX5_SET(fte_match_set_lyr_2_4, headers_c, udp_sport,
967 ntohs(enc_ports.mask->src));
968 MLX5_SET(fte_match_set_lyr_2_4, headers_v, udp_sport,
969 ntohs(enc_ports.key->src));
970
971 return 0;
972 }
973
