1 // SPDX-License-Identifier: GPL-2.0-only
2 #include <linux/module.h>
3 #include <linux/errno.h>
4 #include <linux/socket.h>
5 #include <linux/skbuff.h>
6 #include <linux/ip.h>
7 #include <linux/icmp.h>
8 #include <linux/udp.h>
9 #include <linux/types.h>
10 #include <linux/kernel.h>
11 #include <net/genetlink.h>
12 #include <net/gro.h>
13 #include <net/gue.h>
14 #include <net/fou.h>
15 #include <net/ip.h>
16 #include <net/protocol.h>
17 #include <net/udp.h>
18 #include <net/udp_tunnel.h>
19 #include <uapi/linux/fou.h>
20 #include <uapi/linux/genetlink.h>
21
22 struct fou {
23 struct socket *sock;
24 u8 protocol;
25 u8 flags;
26 __be16 port;
27 u8 family;
28 u16 type;
29 struct list_head list;
30 struct rcu_head rcu;
31 };
32
33 #define FOU_F_REMCSUM_NOPARTIAL BIT(0)
34
35 struct fou_cfg {
36 u16 type;
37 u8 protocol;
38 u8 flags;
39 struct udp_port_cfg udp_config;
40 };
41
42 static unsigned int fou_net_id;
43
44 struct fou_net {
45 struct list_head fou_list;
46 struct mutex fou_lock;
47 };
48
fou_from_sock(struct sock * sk)49 static inline struct fou *fou_from_sock(struct sock *sk)
50 {
51 return sk->sk_user_data;
52 }
53
fou_recv_pull(struct sk_buff * skb,struct fou * fou,size_t len)54 static int fou_recv_pull(struct sk_buff *skb, struct fou *fou, size_t len)
55 {
56 /* Remove 'len' bytes from the packet (UDP header and
57 * FOU header if present).
58 */
59 if (fou->family == AF_INET)
60 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
61 else
62 ipv6_hdr(skb)->payload_len =
63 htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
64
65 __skb_pull(skb, len);
66 skb_postpull_rcsum(skb, udp_hdr(skb), len);
67 skb_reset_transport_header(skb);
68 return iptunnel_pull_offloads(skb);
69 }
70
fou_udp_recv(struct sock * sk,struct sk_buff * skb)71 static int fou_udp_recv(struct sock *sk, struct sk_buff *skb)
72 {
73 struct fou *fou = fou_from_sock(sk);
74
75 if (!fou)
76 return 1;
77
78 if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
79 goto drop;
80
81 return -fou->protocol;
82
83 drop:
84 kfree_skb(skb);
85 return 0;
86 }
87
gue_remcsum(struct sk_buff * skb,struct guehdr * guehdr,void * data,size_t hdrlen,u8 ipproto,bool nopartial)88 static struct guehdr *gue_remcsum(struct sk_buff *skb, struct guehdr *guehdr,
89 void *data, size_t hdrlen, u8 ipproto,
90 bool nopartial)
91 {
92 __be16 *pd = data;
93 size_t start = ntohs(pd[0]);
94 size_t offset = ntohs(pd[1]);
95 size_t plen = sizeof(struct udphdr) + hdrlen +
96 max_t(size_t, offset + sizeof(u16), start);
97
98 if (skb->remcsum_offload)
99 return guehdr;
100
101 if (!pskb_may_pull(skb, plen))
102 return NULL;
103 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
104
105 skb_remcsum_process(skb, (void *)guehdr + hdrlen,
106 start, offset, nopartial);
107
108 return guehdr;
109 }
110
gue_control_message(struct sk_buff * skb,struct guehdr * guehdr)111 static int gue_control_message(struct sk_buff *skb, struct guehdr *guehdr)
112 {
113 /* No support yet */
114 kfree_skb(skb);
115 return 0;
116 }
117
gue_udp_recv(struct sock * sk,struct sk_buff * skb)118 static int gue_udp_recv(struct sock *sk, struct sk_buff *skb)
119 {
120 struct fou *fou = fou_from_sock(sk);
121 size_t len, optlen, hdrlen;
122 struct guehdr *guehdr;
123 void *data;
124 u16 doffset = 0;
125 u8 proto_ctype;
126
127 if (!fou)
128 return 1;
129
130 len = sizeof(struct udphdr) + sizeof(struct guehdr);
131 if (!pskb_may_pull(skb, len))
132 goto drop;
133
134 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
135
136 switch (guehdr->version) {
137 case 0: /* Full GUE header present */
138 break;
139
140 case 1: {
141 /* Direct encapsulation of IPv4 or IPv6 */
142
143 int prot;
144
145 switch (((struct iphdr *)guehdr)->version) {
146 case 4:
147 prot = IPPROTO_IPIP;
148 break;
149 case 6:
150 prot = IPPROTO_IPV6;
151 break;
152 default:
153 goto drop;
154 }
155
156 if (fou_recv_pull(skb, fou, sizeof(struct udphdr)))
157 goto drop;
158
159 return -prot;
160 }
161
162 default: /* Undefined version */
163 goto drop;
164 }
165
166 optlen = guehdr->hlen << 2;
167 len += optlen;
168
169 if (!pskb_may_pull(skb, len))
170 goto drop;
171
172 /* guehdr may change after pull */
173 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
174
175 if (validate_gue_flags(guehdr, optlen))
176 goto drop;
177
178 hdrlen = sizeof(struct guehdr) + optlen;
179
180 if (fou->family == AF_INET)
181 ip_hdr(skb)->tot_len = htons(ntohs(ip_hdr(skb)->tot_len) - len);
182 else
183 ipv6_hdr(skb)->payload_len =
184 htons(ntohs(ipv6_hdr(skb)->payload_len) - len);
185
186 /* Pull csum through the guehdr now . This can be used if
187 * there is a remote checksum offload.
188 */
189 skb_postpull_rcsum(skb, udp_hdr(skb), len);
190
191 data = &guehdr[1];
192
193 if (guehdr->flags & GUE_FLAG_PRIV) {
194 __be32 flags = *(__be32 *)(data + doffset);
195
196 doffset += GUE_LEN_PRIV;
197
198 if (flags & GUE_PFLAG_REMCSUM) {
199 guehdr = gue_remcsum(skb, guehdr, data + doffset,
200 hdrlen, guehdr->proto_ctype,
201 !!(fou->flags &
202 FOU_F_REMCSUM_NOPARTIAL));
203 if (!guehdr)
204 goto drop;
205
206 data = &guehdr[1];
207
208 doffset += GUE_PLEN_REMCSUM;
209 }
210 }
211
212 if (unlikely(guehdr->control))
213 return gue_control_message(skb, guehdr);
214
215 proto_ctype = guehdr->proto_ctype;
216 __skb_pull(skb, sizeof(struct udphdr) + hdrlen);
217 skb_reset_transport_header(skb);
218
219 if (iptunnel_pull_offloads(skb))
220 goto drop;
221
222 return -proto_ctype;
223
224 drop:
225 kfree_skb(skb);
226 return 0;
227 }
228
fou_gro_receive(struct sock * sk,struct list_head * head,struct sk_buff * skb)229 static struct sk_buff *fou_gro_receive(struct sock *sk,
230 struct list_head *head,
231 struct sk_buff *skb)
232 {
233 const struct net_offload __rcu **offloads;
234 u8 proto = fou_from_sock(sk)->protocol;
235 const struct net_offload *ops;
236 struct sk_buff *pp = NULL;
237
238 /* We can clear the encap_mark for FOU as we are essentially doing
239 * one of two possible things. We are either adding an L4 tunnel
240 * header to the outer L3 tunnel header, or we are simply
241 * treating the GRE tunnel header as though it is a UDP protocol
242 * specific header such as VXLAN or GENEVE.
243 */
244 NAPI_GRO_CB(skb)->encap_mark = 0;
245
246 /* Flag this frame as already having an outer encap header */
247 NAPI_GRO_CB(skb)->is_fou = 1;
248
249 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
250 ops = rcu_dereference(offloads[proto]);
251 if (!ops || !ops->callbacks.gro_receive)
252 goto out;
253
254 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
255
256 out:
257 return pp;
258 }
259
fou_gro_complete(struct sock * sk,struct sk_buff * skb,int nhoff)260 static int fou_gro_complete(struct sock *sk, struct sk_buff *skb,
261 int nhoff)
262 {
263 const struct net_offload __rcu **offloads;
264 u8 proto = fou_from_sock(sk)->protocol;
265 const struct net_offload *ops;
266 int err = -ENOSYS;
267
268 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
269 ops = rcu_dereference(offloads[proto]);
270 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
271 goto out;
272
273 err = ops->callbacks.gro_complete(skb, nhoff);
274
275 skb_set_inner_mac_header(skb, nhoff);
276
277 out:
278 return err;
279 }
280
gue_gro_remcsum(struct sk_buff * skb,unsigned int off,struct guehdr * guehdr,void * data,size_t hdrlen,struct gro_remcsum * grc,bool nopartial)281 static struct guehdr *gue_gro_remcsum(struct sk_buff *skb, unsigned int off,
282 struct guehdr *guehdr, void *data,
283 size_t hdrlen, struct gro_remcsum *grc,
284 bool nopartial)
285 {
286 __be16 *pd = data;
287 size_t start = ntohs(pd[0]);
288 size_t offset = ntohs(pd[1]);
289
290 if (skb->remcsum_offload)
291 return guehdr;
292
293 if (!NAPI_GRO_CB(skb)->csum_valid)
294 return NULL;
295
296 guehdr = skb_gro_remcsum_process(skb, (void *)guehdr, off, hdrlen,
297 start, offset, grc, nopartial);
298
299 skb->remcsum_offload = 1;
300
301 return guehdr;
302 }
303
gue_gro_receive(struct sock * sk,struct list_head * head,struct sk_buff * skb)304 static struct sk_buff *gue_gro_receive(struct sock *sk,
305 struct list_head *head,
306 struct sk_buff *skb)
307 {
308 const struct net_offload __rcu **offloads;
309 const struct net_offload *ops;
310 struct sk_buff *pp = NULL;
311 struct sk_buff *p;
312 struct guehdr *guehdr;
313 size_t len, optlen, hdrlen, off;
314 void *data;
315 u16 doffset = 0;
316 int flush = 1;
317 struct fou *fou = fou_from_sock(sk);
318 struct gro_remcsum grc;
319 u8 proto;
320
321 skb_gro_remcsum_init(&grc);
322
323 off = skb_gro_offset(skb);
324 len = off + sizeof(*guehdr);
325
326 guehdr = skb_gro_header(skb, len, off);
327 if (unlikely(!guehdr))
328 goto out;
329
330 switch (guehdr->version) {
331 case 0:
332 break;
333 case 1:
334 switch (((struct iphdr *)guehdr)->version) {
335 case 4:
336 proto = IPPROTO_IPIP;
337 break;
338 case 6:
339 proto = IPPROTO_IPV6;
340 break;
341 default:
342 goto out;
343 }
344 goto next_proto;
345 default:
346 goto out;
347 }
348
349 optlen = guehdr->hlen << 2;
350 len += optlen;
351
352 if (skb_gro_header_hard(skb, len)) {
353 guehdr = skb_gro_header_slow(skb, len, off);
354 if (unlikely(!guehdr))
355 goto out;
356 }
357
358 if (unlikely(guehdr->control) || guehdr->version != 0 ||
359 validate_gue_flags(guehdr, optlen))
360 goto out;
361
362 hdrlen = sizeof(*guehdr) + optlen;
363
364 /* Adjust NAPI_GRO_CB(skb)->csum to account for guehdr,
365 * this is needed if there is a remote checkcsum offload.
366 */
367 skb_gro_postpull_rcsum(skb, guehdr, hdrlen);
368
369 data = &guehdr[1];
370
371 if (guehdr->flags & GUE_FLAG_PRIV) {
372 __be32 flags = *(__be32 *)(data + doffset);
373
374 doffset += GUE_LEN_PRIV;
375
376 if (flags & GUE_PFLAG_REMCSUM) {
377 guehdr = gue_gro_remcsum(skb, off, guehdr,
378 data + doffset, hdrlen, &grc,
379 !!(fou->flags &
380 FOU_F_REMCSUM_NOPARTIAL));
381
382 if (!guehdr)
383 goto out;
384
385 data = &guehdr[1];
386
387 doffset += GUE_PLEN_REMCSUM;
388 }
389 }
390
391 skb_gro_pull(skb, hdrlen);
392
393 list_for_each_entry(p, head, list) {
394 const struct guehdr *guehdr2;
395
396 if (!NAPI_GRO_CB(p)->same_flow)
397 continue;
398
399 guehdr2 = (struct guehdr *)(p->data + off);
400
401 /* Compare base GUE header to be equal (covers
402 * hlen, version, proto_ctype, and flags.
403 */
404 if (guehdr->word != guehdr2->word) {
405 NAPI_GRO_CB(p)->same_flow = 0;
406 continue;
407 }
408
409 /* Compare optional fields are the same. */
410 if (guehdr->hlen && memcmp(&guehdr[1], &guehdr2[1],
411 guehdr->hlen << 2)) {
412 NAPI_GRO_CB(p)->same_flow = 0;
413 continue;
414 }
415 }
416
417 proto = guehdr->proto_ctype;
418
419 next_proto:
420
421 /* We can clear the encap_mark for GUE as we are essentially doing
422 * one of two possible things. We are either adding an L4 tunnel
423 * header to the outer L3 tunnel header, or we are simply
424 * treating the GRE tunnel header as though it is a UDP protocol
425 * specific header such as VXLAN or GENEVE.
426 */
427 NAPI_GRO_CB(skb)->encap_mark = 0;
428
429 /* Flag this frame as already having an outer encap header */
430 NAPI_GRO_CB(skb)->is_fou = 1;
431
432 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
433 ops = rcu_dereference(offloads[proto]);
434 if (WARN_ON_ONCE(!ops || !ops->callbacks.gro_receive))
435 goto out;
436
437 pp = call_gro_receive(ops->callbacks.gro_receive, head, skb);
438 flush = 0;
439
440 out:
441 skb_gro_flush_final_remcsum(skb, pp, flush, &grc);
442
443 return pp;
444 }
445
gue_gro_complete(struct sock * sk,struct sk_buff * skb,int nhoff)446 static int gue_gro_complete(struct sock *sk, struct sk_buff *skb, int nhoff)
447 {
448 struct guehdr *guehdr = (struct guehdr *)(skb->data + nhoff);
449 const struct net_offload __rcu **offloads;
450 const struct net_offload *ops;
451 unsigned int guehlen = 0;
452 u8 proto;
453 int err = -ENOENT;
454
455 switch (guehdr->version) {
456 case 0:
457 proto = guehdr->proto_ctype;
458 guehlen = sizeof(*guehdr) + (guehdr->hlen << 2);
459 break;
460 case 1:
461 switch (((struct iphdr *)guehdr)->version) {
462 case 4:
463 proto = IPPROTO_IPIP;
464 break;
465 case 6:
466 proto = IPPROTO_IPV6;
467 break;
468 default:
469 return err;
470 }
471 break;
472 default:
473 return err;
474 }
475
476 offloads = NAPI_GRO_CB(skb)->is_ipv6 ? inet6_offloads : inet_offloads;
477 ops = rcu_dereference(offloads[proto]);
478 if (WARN_ON(!ops || !ops->callbacks.gro_complete))
479 goto out;
480
481 err = ops->callbacks.gro_complete(skb, nhoff + guehlen);
482
483 skb_set_inner_mac_header(skb, nhoff + guehlen);
484
485 out:
486 return err;
487 }
488
fou_cfg_cmp(struct fou * fou,struct fou_cfg * cfg)489 static bool fou_cfg_cmp(struct fou *fou, struct fou_cfg *cfg)
490 {
491 struct sock *sk = fou->sock->sk;
492 struct udp_port_cfg *udp_cfg = &cfg->udp_config;
493
494 if (fou->family != udp_cfg->family ||
495 fou->port != udp_cfg->local_udp_port ||
496 sk->sk_dport != udp_cfg->peer_udp_port ||
497 sk->sk_bound_dev_if != udp_cfg->bind_ifindex)
498 return false;
499
500 if (fou->family == AF_INET) {
501 if (sk->sk_rcv_saddr != udp_cfg->local_ip.s_addr ||
502 sk->sk_daddr != udp_cfg->peer_ip.s_addr)
503 return false;
504 else
505 return true;
506 #if IS_ENABLED(CONFIG_IPV6)
507 } else {
508 if (ipv6_addr_cmp(&sk->sk_v6_rcv_saddr, &udp_cfg->local_ip6) ||
509 ipv6_addr_cmp(&sk->sk_v6_daddr, &udp_cfg->peer_ip6))
510 return false;
511 else
512 return true;
513 #endif
514 }
515
516 return false;
517 }
518
fou_add_to_port_list(struct net * net,struct fou * fou,struct fou_cfg * cfg)519 static int fou_add_to_port_list(struct net *net, struct fou *fou,
520 struct fou_cfg *cfg)
521 {
522 struct fou_net *fn = net_generic(net, fou_net_id);
523 struct fou *fout;
524
525 mutex_lock(&fn->fou_lock);
526 list_for_each_entry(fout, &fn->fou_list, list) {
527 if (fou_cfg_cmp(fout, cfg)) {
528 mutex_unlock(&fn->fou_lock);
529 return -EALREADY;
530 }
531 }
532
533 list_add(&fou->list, &fn->fou_list);
534 mutex_unlock(&fn->fou_lock);
535
536 return 0;
537 }
538
fou_release(struct fou * fou)539 static void fou_release(struct fou *fou)
540 {
541 struct socket *sock = fou->sock;
542
543 list_del(&fou->list);
544 udp_tunnel_sock_release(sock);
545
546 kfree_rcu(fou, rcu);
547 }
548
fou_create(struct net * net,struct fou_cfg * cfg,struct socket ** sockp)549 static int fou_create(struct net *net, struct fou_cfg *cfg,
550 struct socket **sockp)
551 {
552 struct socket *sock = NULL;
553 struct fou *fou = NULL;
554 struct sock *sk;
555 struct udp_tunnel_sock_cfg tunnel_cfg;
556 int err;
557
558 /* Open UDP socket */
559 err = udp_sock_create(net, &cfg->udp_config, &sock);
560 if (err < 0)
561 goto error;
562
563 /* Allocate FOU port structure */
564 fou = kzalloc(sizeof(*fou), GFP_KERNEL);
565 if (!fou) {
566 err = -ENOMEM;
567 goto error;
568 }
569
570 sk = sock->sk;
571
572 fou->port = cfg->udp_config.local_udp_port;
573 fou->family = cfg->udp_config.family;
574 fou->flags = cfg->flags;
575 fou->type = cfg->type;
576 fou->sock = sock;
577
578 memset(&tunnel_cfg, 0, sizeof(tunnel_cfg));
579 tunnel_cfg.encap_type = 1;
580 tunnel_cfg.sk_user_data = fou;
581 tunnel_cfg.encap_destroy = NULL;
582
583 /* Initial for fou type */
584 switch (cfg->type) {
585 case FOU_ENCAP_DIRECT:
586 tunnel_cfg.encap_rcv = fou_udp_recv;
587 tunnel_cfg.gro_receive = fou_gro_receive;
588 tunnel_cfg.gro_complete = fou_gro_complete;
589 fou->protocol = cfg->protocol;
590 break;
591 case FOU_ENCAP_GUE:
592 tunnel_cfg.encap_rcv = gue_udp_recv;
593 tunnel_cfg.gro_receive = gue_gro_receive;
594 tunnel_cfg.gro_complete = gue_gro_complete;
595 break;
596 default:
597 err = -EINVAL;
598 goto error;
599 }
600
601 setup_udp_tunnel_sock(net, sock, &tunnel_cfg);
602
603 sk->sk_allocation = GFP_ATOMIC;
604
605 err = fou_add_to_port_list(net, fou, cfg);
606 if (err)
607 goto error;
608
609 if (sockp)
610 *sockp = sock;
611
612 return 0;
613
614 error:
615 kfree(fou);
616 if (sock)
617 udp_tunnel_sock_release(sock);
618
619 return err;
620 }
621
fou_destroy(struct net * net,struct fou_cfg * cfg)622 static int fou_destroy(struct net *net, struct fou_cfg *cfg)
623 {
624 struct fou_net *fn = net_generic(net, fou_net_id);
625 int err = -EINVAL;
626 struct fou *fou;
627
628 mutex_lock(&fn->fou_lock);
629 list_for_each_entry(fou, &fn->fou_list, list) {
630 if (fou_cfg_cmp(fou, cfg)) {
631 fou_release(fou);
632 err = 0;
633 break;
634 }
635 }
636 mutex_unlock(&fn->fou_lock);
637
638 return err;
639 }
640
641 static struct genl_family fou_nl_family;
642
643 static const struct nla_policy fou_nl_policy[FOU_ATTR_MAX + 1] = {
644 [FOU_ATTR_PORT] = { .type = NLA_U16, },
645 [FOU_ATTR_AF] = { .type = NLA_U8, },
646 [FOU_ATTR_IPPROTO] = { .type = NLA_U8, },
647 [FOU_ATTR_TYPE] = { .type = NLA_U8, },
648 [FOU_ATTR_REMCSUM_NOPARTIAL] = { .type = NLA_FLAG, },
649 [FOU_ATTR_LOCAL_V4] = { .type = NLA_U32, },
650 [FOU_ATTR_PEER_V4] = { .type = NLA_U32, },
651 [FOU_ATTR_LOCAL_V6] = { .len = sizeof(struct in6_addr), },
652 [FOU_ATTR_PEER_V6] = { .len = sizeof(struct in6_addr), },
653 [FOU_ATTR_PEER_PORT] = { .type = NLA_U16, },
654 [FOU_ATTR_IFINDEX] = { .type = NLA_S32, },
655 };
656
parse_nl_config(struct genl_info * info,struct fou_cfg * cfg)657 static int parse_nl_config(struct genl_info *info,
658 struct fou_cfg *cfg)
659 {
660 bool has_local = false, has_peer = false;
661 struct nlattr *attr;
662 int ifindex;
663 __be16 port;
664
665 memset(cfg, 0, sizeof(*cfg));
666
667 cfg->udp_config.family = AF_INET;
668
669 if (info->attrs[FOU_ATTR_AF]) {
670 u8 family = nla_get_u8(info->attrs[FOU_ATTR_AF]);
671
672 switch (family) {
673 case AF_INET:
674 break;
675 case AF_INET6:
676 cfg->udp_config.ipv6_v6only = 1;
677 break;
678 default:
679 return -EAFNOSUPPORT;
680 }
681
682 cfg->udp_config.family = family;
683 }
684
685 if (info->attrs[FOU_ATTR_PORT]) {
686 port = nla_get_be16(info->attrs[FOU_ATTR_PORT]);
687 cfg->udp_config.local_udp_port = port;
688 }
689
690 if (info->attrs[FOU_ATTR_IPPROTO])
691 cfg->protocol = nla_get_u8(info->attrs[FOU_ATTR_IPPROTO]);
692
693 if (info->attrs[FOU_ATTR_TYPE])
694 cfg->type = nla_get_u8(info->attrs[FOU_ATTR_TYPE]);
695
696 if (info->attrs[FOU_ATTR_REMCSUM_NOPARTIAL])
697 cfg->flags |= FOU_F_REMCSUM_NOPARTIAL;
698
699 if (cfg->udp_config.family == AF_INET) {
700 if (info->attrs[FOU_ATTR_LOCAL_V4]) {
701 attr = info->attrs[FOU_ATTR_LOCAL_V4];
702 cfg->udp_config.local_ip.s_addr = nla_get_in_addr(attr);
703 has_local = true;
704 }
705
706 if (info->attrs[FOU_ATTR_PEER_V4]) {
707 attr = info->attrs[FOU_ATTR_PEER_V4];
708 cfg->udp_config.peer_ip.s_addr = nla_get_in_addr(attr);
709 has_peer = true;
710 }
711 #if IS_ENABLED(CONFIG_IPV6)
712 } else {
713 if (info->attrs[FOU_ATTR_LOCAL_V6]) {
714 attr = info->attrs[FOU_ATTR_LOCAL_V6];
715 cfg->udp_config.local_ip6 = nla_get_in6_addr(attr);
716 has_local = true;
717 }
718
719 if (info->attrs[FOU_ATTR_PEER_V6]) {
720 attr = info->attrs[FOU_ATTR_PEER_V6];
721 cfg->udp_config.peer_ip6 = nla_get_in6_addr(attr);
722 has_peer = true;
723 }
724 #endif
725 }
726
727 if (has_peer) {
728 if (info->attrs[FOU_ATTR_PEER_PORT]) {
729 port = nla_get_be16(info->attrs[FOU_ATTR_PEER_PORT]);
730 cfg->udp_config.peer_udp_port = port;
731 } else {
732 return -EINVAL;
733 }
734 }
735
736 if (info->attrs[FOU_ATTR_IFINDEX]) {
737 if (!has_local)
738 return -EINVAL;
739
740 ifindex = nla_get_s32(info->attrs[FOU_ATTR_IFINDEX]);
741
742 cfg->udp_config.bind_ifindex = ifindex;
743 }
744
745 return 0;
746 }
747
fou_nl_cmd_add_port(struct sk_buff * skb,struct genl_info * info)748 static int fou_nl_cmd_add_port(struct sk_buff *skb, struct genl_info *info)
749 {
750 struct net *net = genl_info_net(info);
751 struct fou_cfg cfg;
752 int err;
753
754 err = parse_nl_config(info, &cfg);
755 if (err)
756 return err;
757
758 return fou_create(net, &cfg, NULL);
759 }
760
fou_nl_cmd_rm_port(struct sk_buff * skb,struct genl_info * info)761 static int fou_nl_cmd_rm_port(struct sk_buff *skb, struct genl_info *info)
762 {
763 struct net *net = genl_info_net(info);
764 struct fou_cfg cfg;
765 int err;
766
767 err = parse_nl_config(info, &cfg);
768 if (err)
769 return err;
770
771 return fou_destroy(net, &cfg);
772 }
773
fou_fill_info(struct fou * fou,struct sk_buff * msg)774 static int fou_fill_info(struct fou *fou, struct sk_buff *msg)
775 {
776 struct sock *sk = fou->sock->sk;
777
778 if (nla_put_u8(msg, FOU_ATTR_AF, fou->sock->sk->sk_family) ||
779 nla_put_be16(msg, FOU_ATTR_PORT, fou->port) ||
780 nla_put_be16(msg, FOU_ATTR_PEER_PORT, sk->sk_dport) ||
781 nla_put_u8(msg, FOU_ATTR_IPPROTO, fou->protocol) ||
782 nla_put_u8(msg, FOU_ATTR_TYPE, fou->type) ||
783 nla_put_s32(msg, FOU_ATTR_IFINDEX, sk->sk_bound_dev_if))
784 return -1;
785
786 if (fou->flags & FOU_F_REMCSUM_NOPARTIAL)
787 if (nla_put_flag(msg, FOU_ATTR_REMCSUM_NOPARTIAL))
788 return -1;
789
790 if (fou->sock->sk->sk_family == AF_INET) {
791 if (nla_put_in_addr(msg, FOU_ATTR_LOCAL_V4, sk->sk_rcv_saddr))
792 return -1;
793
794 if (nla_put_in_addr(msg, FOU_ATTR_PEER_V4, sk->sk_daddr))
795 return -1;
796 #if IS_ENABLED(CONFIG_IPV6)
797 } else {
798 if (nla_put_in6_addr(msg, FOU_ATTR_LOCAL_V6,
799 &sk->sk_v6_rcv_saddr))
800 return -1;
801
802 if (nla_put_in6_addr(msg, FOU_ATTR_PEER_V6, &sk->sk_v6_daddr))
803 return -1;
804 #endif
805 }
806
807 return 0;
808 }
809
fou_dump_info(struct fou * fou,u32 portid,u32 seq,u32 flags,struct sk_buff * skb,u8 cmd)810 static int fou_dump_info(struct fou *fou, u32 portid, u32 seq,
811 u32 flags, struct sk_buff *skb, u8 cmd)
812 {
813 void *hdr;
814
815 hdr = genlmsg_put(skb, portid, seq, &fou_nl_family, flags, cmd);
816 if (!hdr)
817 return -ENOMEM;
818
819 if (fou_fill_info(fou, skb) < 0)
820 goto nla_put_failure;
821
822 genlmsg_end(skb, hdr);
823 return 0;
824
825 nla_put_failure:
826 genlmsg_cancel(skb, hdr);
827 return -EMSGSIZE;
828 }
829
fou_nl_cmd_get_port(struct sk_buff * skb,struct genl_info * info)830 static int fou_nl_cmd_get_port(struct sk_buff *skb, struct genl_info *info)
831 {
832 struct net *net = genl_info_net(info);
833 struct fou_net *fn = net_generic(net, fou_net_id);
834 struct sk_buff *msg;
835 struct fou_cfg cfg;
836 struct fou *fout;
837 __be16 port;
838 u8 family;
839 int ret;
840
841 ret = parse_nl_config(info, &cfg);
842 if (ret)
843 return ret;
844 port = cfg.udp_config.local_udp_port;
845 if (port == 0)
846 return -EINVAL;
847
848 family = cfg.udp_config.family;
849 if (family != AF_INET && family != AF_INET6)
850 return -EINVAL;
851
852 msg = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
853 if (!msg)
854 return -ENOMEM;
855
856 ret = -ESRCH;
857 mutex_lock(&fn->fou_lock);
858 list_for_each_entry(fout, &fn->fou_list, list) {
859 if (fou_cfg_cmp(fout, &cfg)) {
860 ret = fou_dump_info(fout, info->snd_portid,
861 info->snd_seq, 0, msg,
862 info->genlhdr->cmd);
863 break;
864 }
865 }
866 mutex_unlock(&fn->fou_lock);
867 if (ret < 0)
868 goto out_free;
869
870 return genlmsg_reply(msg, info);
871
872 out_free:
873 nlmsg_free(msg);
874 return ret;
875 }
876
fou_nl_dump(struct sk_buff * skb,struct netlink_callback * cb)877 static int fou_nl_dump(struct sk_buff *skb, struct netlink_callback *cb)
878 {
879 struct net *net = sock_net(skb->sk);
880 struct fou_net *fn = net_generic(net, fou_net_id);
881 struct fou *fout;
882 int idx = 0, ret;
883
884 mutex_lock(&fn->fou_lock);
885 list_for_each_entry(fout, &fn->fou_list, list) {
886 if (idx++ < cb->args[0])
887 continue;
888 ret = fou_dump_info(fout, NETLINK_CB(cb->skb).portid,
889 cb->nlh->nlmsg_seq, NLM_F_MULTI,
890 skb, FOU_CMD_GET);
891 if (ret)
892 break;
893 }
894 mutex_unlock(&fn->fou_lock);
895
896 cb->args[0] = idx;
897 return skb->len;
898 }
899
900 static const struct genl_small_ops fou_nl_ops[] = {
901 {
902 .cmd = FOU_CMD_ADD,
903 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
904 .doit = fou_nl_cmd_add_port,
905 .flags = GENL_ADMIN_PERM,
906 },
907 {
908 .cmd = FOU_CMD_DEL,
909 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
910 .doit = fou_nl_cmd_rm_port,
911 .flags = GENL_ADMIN_PERM,
912 },
913 {
914 .cmd = FOU_CMD_GET,
915 .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
916 .doit = fou_nl_cmd_get_port,
917 .dumpit = fou_nl_dump,
918 },
919 };
920
921 static struct genl_family fou_nl_family __ro_after_init = {
922 .hdrsize = 0,
923 .name = FOU_GENL_NAME,
924 .version = FOU_GENL_VERSION,
925 .maxattr = FOU_ATTR_MAX,
926 .policy = fou_nl_policy,
927 .netnsok = true,
928 .module = THIS_MODULE,
929 .small_ops = fou_nl_ops,
930 .n_small_ops = ARRAY_SIZE(fou_nl_ops),
931 .resv_start_op = FOU_CMD_GET + 1,
932 };
933
fou_encap_hlen(struct ip_tunnel_encap * e)934 size_t fou_encap_hlen(struct ip_tunnel_encap *e)
935 {
936 return sizeof(struct udphdr);
937 }
938 EXPORT_SYMBOL(fou_encap_hlen);
939
gue_encap_hlen(struct ip_tunnel_encap * e)940 size_t gue_encap_hlen(struct ip_tunnel_encap *e)
941 {
942 size_t len;
943 bool need_priv = false;
944
945 len = sizeof(struct udphdr) + sizeof(struct guehdr);
946
947 if (e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) {
948 len += GUE_PLEN_REMCSUM;
949 need_priv = true;
950 }
951
952 len += need_priv ? GUE_LEN_PRIV : 0;
953
954 return len;
955 }
956 EXPORT_SYMBOL(gue_encap_hlen);
957
__fou_build_header(struct sk_buff * skb,struct ip_tunnel_encap * e,u8 * protocol,__be16 * sport,int type)958 int __fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
959 u8 *protocol, __be16 *sport, int type)
960 {
961 int err;
962
963 err = iptunnel_handle_offloads(skb, type);
964 if (err)
965 return err;
966
967 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
968 skb, 0, 0, false);
969
970 return 0;
971 }
972 EXPORT_SYMBOL(__fou_build_header);
973
__gue_build_header(struct sk_buff * skb,struct ip_tunnel_encap * e,u8 * protocol,__be16 * sport,int type)974 int __gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
975 u8 *protocol, __be16 *sport, int type)
976 {
977 struct guehdr *guehdr;
978 size_t hdrlen, optlen = 0;
979 void *data;
980 bool need_priv = false;
981 int err;
982
983 if ((e->flags & TUNNEL_ENCAP_FLAG_REMCSUM) &&
984 skb->ip_summed == CHECKSUM_PARTIAL) {
985 optlen += GUE_PLEN_REMCSUM;
986 type |= SKB_GSO_TUNNEL_REMCSUM;
987 need_priv = true;
988 }
989
990 optlen += need_priv ? GUE_LEN_PRIV : 0;
991
992 err = iptunnel_handle_offloads(skb, type);
993 if (err)
994 return err;
995
996 /* Get source port (based on flow hash) before skb_push */
997 *sport = e->sport ? : udp_flow_src_port(dev_net(skb->dev),
998 skb, 0, 0, false);
999
1000 hdrlen = sizeof(struct guehdr) + optlen;
1001
1002 skb_push(skb, hdrlen);
1003
1004 guehdr = (struct guehdr *)skb->data;
1005
1006 guehdr->control = 0;
1007 guehdr->version = 0;
1008 guehdr->hlen = optlen >> 2;
1009 guehdr->flags = 0;
1010 guehdr->proto_ctype = *protocol;
1011
1012 data = &guehdr[1];
1013
1014 if (need_priv) {
1015 __be32 *flags = data;
1016
1017 guehdr->flags |= GUE_FLAG_PRIV;
1018 *flags = 0;
1019 data += GUE_LEN_PRIV;
1020
1021 if (type & SKB_GSO_TUNNEL_REMCSUM) {
1022 u16 csum_start = skb_checksum_start_offset(skb);
1023 __be16 *pd = data;
1024
1025 if (csum_start < hdrlen)
1026 return -EINVAL;
1027
1028 csum_start -= hdrlen;
1029 pd[0] = htons(csum_start);
1030 pd[1] = htons(csum_start + skb->csum_offset);
1031
1032 if (!skb_is_gso(skb)) {
1033 skb->ip_summed = CHECKSUM_NONE;
1034 skb->encapsulation = 0;
1035 }
1036
1037 *flags |= GUE_PFLAG_REMCSUM;
1038 data += GUE_PLEN_REMCSUM;
1039 }
1040
1041 }
1042
1043 return 0;
1044 }
1045 EXPORT_SYMBOL(__gue_build_header);
1046
1047 #ifdef CONFIG_NET_FOU_IP_TUNNELS
1048
fou_build_udp(struct sk_buff * skb,struct ip_tunnel_encap * e,struct flowi4 * fl4,u8 * protocol,__be16 sport)1049 static void fou_build_udp(struct sk_buff *skb, struct ip_tunnel_encap *e,
1050 struct flowi4 *fl4, u8 *protocol, __be16 sport)
1051 {
1052 struct udphdr *uh;
1053
1054 skb_push(skb, sizeof(struct udphdr));
1055 skb_reset_transport_header(skb);
1056
1057 uh = udp_hdr(skb);
1058
1059 uh->dest = e->dport;
1060 uh->source = sport;
1061 uh->len = htons(skb->len);
1062 udp_set_csum(!(e->flags & TUNNEL_ENCAP_FLAG_CSUM), skb,
1063 fl4->saddr, fl4->daddr, skb->len);
1064
1065 *protocol = IPPROTO_UDP;
1066 }
1067
fou_build_header(struct sk_buff * skb,struct ip_tunnel_encap * e,u8 * protocol,struct flowi4 * fl4)1068 static int fou_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1069 u8 *protocol, struct flowi4 *fl4)
1070 {
1071 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1072 SKB_GSO_UDP_TUNNEL;
1073 __be16 sport;
1074 int err;
1075
1076 err = __fou_build_header(skb, e, protocol, &sport, type);
1077 if (err)
1078 return err;
1079
1080 fou_build_udp(skb, e, fl4, protocol, sport);
1081
1082 return 0;
1083 }
1084
gue_build_header(struct sk_buff * skb,struct ip_tunnel_encap * e,u8 * protocol,struct flowi4 * fl4)1085 static int gue_build_header(struct sk_buff *skb, struct ip_tunnel_encap *e,
1086 u8 *protocol, struct flowi4 *fl4)
1087 {
1088 int type = e->flags & TUNNEL_ENCAP_FLAG_CSUM ? SKB_GSO_UDP_TUNNEL_CSUM :
1089 SKB_GSO_UDP_TUNNEL;
1090 __be16 sport;
1091 int err;
1092
1093 err = __gue_build_header(skb, e, protocol, &sport, type);
1094 if (err)
1095 return err;
1096
1097 fou_build_udp(skb, e, fl4, protocol, sport);
1098
1099 return 0;
1100 }
1101
gue_err_proto_handler(int proto,struct sk_buff * skb,u32 info)1102 static int gue_err_proto_handler(int proto, struct sk_buff *skb, u32 info)
1103 {
1104 const struct net_protocol *ipprot = rcu_dereference(inet_protos[proto]);
1105
1106 if (ipprot && ipprot->err_handler) {
1107 if (!ipprot->err_handler(skb, info))
1108 return 0;
1109 }
1110
1111 return -ENOENT;
1112 }
1113
gue_err(struct sk_buff * skb,u32 info)1114 static int gue_err(struct sk_buff *skb, u32 info)
1115 {
1116 int transport_offset = skb_transport_offset(skb);
1117 struct guehdr *guehdr;
1118 size_t len, optlen;
1119 int ret;
1120
1121 len = sizeof(struct udphdr) + sizeof(struct guehdr);
1122 if (!pskb_may_pull(skb, transport_offset + len))
1123 return -EINVAL;
1124
1125 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1126
1127 switch (guehdr->version) {
1128 case 0: /* Full GUE header present */
1129 break;
1130 case 1: {
1131 /* Direct encapsulation of IPv4 or IPv6 */
1132 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1133
1134 switch (((struct iphdr *)guehdr)->version) {
1135 case 4:
1136 ret = gue_err_proto_handler(IPPROTO_IPIP, skb, info);
1137 goto out;
1138 #if IS_ENABLED(CONFIG_IPV6)
1139 case 6:
1140 ret = gue_err_proto_handler(IPPROTO_IPV6, skb, info);
1141 goto out;
1142 #endif
1143 default:
1144 ret = -EOPNOTSUPP;
1145 goto out;
1146 }
1147 }
1148 default: /* Undefined version */
1149 return -EOPNOTSUPP;
1150 }
1151
1152 if (guehdr->control)
1153 return -ENOENT;
1154
1155 optlen = guehdr->hlen << 2;
1156
1157 if (!pskb_may_pull(skb, transport_offset + len + optlen))
1158 return -EINVAL;
1159
1160 guehdr = (struct guehdr *)&udp_hdr(skb)[1];
1161 if (validate_gue_flags(guehdr, optlen))
1162 return -EINVAL;
1163
1164 /* Handling exceptions for direct UDP encapsulation in GUE would lead to
1165 * recursion. Besides, this kind of encapsulation can't even be
1166 * configured currently. Discard this.
1167 */
1168 if (guehdr->proto_ctype == IPPROTO_UDP ||
1169 guehdr->proto_ctype == IPPROTO_UDPLITE)
1170 return -EOPNOTSUPP;
1171
1172 skb_set_transport_header(skb, -(int)sizeof(struct icmphdr));
1173 ret = gue_err_proto_handler(guehdr->proto_ctype, skb, info);
1174
1175 out:
1176 skb_set_transport_header(skb, transport_offset);
1177 return ret;
1178 }
1179
1180
1181 static const struct ip_tunnel_encap_ops fou_iptun_ops = {
1182 .encap_hlen = fou_encap_hlen,
1183 .build_header = fou_build_header,
1184 .err_handler = gue_err,
1185 };
1186
1187 static const struct ip_tunnel_encap_ops gue_iptun_ops = {
1188 .encap_hlen = gue_encap_hlen,
1189 .build_header = gue_build_header,
1190 .err_handler = gue_err,
1191 };
1192
ip_tunnel_encap_add_fou_ops(void)1193 static int ip_tunnel_encap_add_fou_ops(void)
1194 {
1195 int ret;
1196
1197 ret = ip_tunnel_encap_add_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1198 if (ret < 0) {
1199 pr_err("can't add fou ops\n");
1200 return ret;
1201 }
1202
1203 ret = ip_tunnel_encap_add_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1204 if (ret < 0) {
1205 pr_err("can't add gue ops\n");
1206 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1207 return ret;
1208 }
1209
1210 return 0;
1211 }
1212
ip_tunnel_encap_del_fou_ops(void)1213 static void ip_tunnel_encap_del_fou_ops(void)
1214 {
1215 ip_tunnel_encap_del_ops(&fou_iptun_ops, TUNNEL_ENCAP_FOU);
1216 ip_tunnel_encap_del_ops(&gue_iptun_ops, TUNNEL_ENCAP_GUE);
1217 }
1218
1219 #else
1220
ip_tunnel_encap_add_fou_ops(void)1221 static int ip_tunnel_encap_add_fou_ops(void)
1222 {
1223 return 0;
1224 }
1225
ip_tunnel_encap_del_fou_ops(void)1226 static void ip_tunnel_encap_del_fou_ops(void)
1227 {
1228 }
1229
1230 #endif
1231
fou_init_net(struct net * net)1232 static __net_init int fou_init_net(struct net *net)
1233 {
1234 struct fou_net *fn = net_generic(net, fou_net_id);
1235
1236 INIT_LIST_HEAD(&fn->fou_list);
1237 mutex_init(&fn->fou_lock);
1238 return 0;
1239 }
1240
fou_exit_net(struct net * net)1241 static __net_exit void fou_exit_net(struct net *net)
1242 {
1243 struct fou_net *fn = net_generic(net, fou_net_id);
1244 struct fou *fou, *next;
1245
1246 /* Close all the FOU sockets */
1247 mutex_lock(&fn->fou_lock);
1248 list_for_each_entry_safe(fou, next, &fn->fou_list, list)
1249 fou_release(fou);
1250 mutex_unlock(&fn->fou_lock);
1251 }
1252
1253 static struct pernet_operations fou_net_ops = {
1254 .init = fou_init_net,
1255 .exit = fou_exit_net,
1256 .id = &fou_net_id,
1257 .size = sizeof(struct fou_net),
1258 };
1259
fou_init(void)1260 static int __init fou_init(void)
1261 {
1262 int ret;
1263
1264 ret = register_pernet_device(&fou_net_ops);
1265 if (ret)
1266 goto exit;
1267
1268 ret = genl_register_family(&fou_nl_family);
1269 if (ret < 0)
1270 goto unregister;
1271
1272 ret = ip_tunnel_encap_add_fou_ops();
1273 if (ret == 0)
1274 return 0;
1275
1276 genl_unregister_family(&fou_nl_family);
1277 unregister:
1278 unregister_pernet_device(&fou_net_ops);
1279 exit:
1280 return ret;
1281 }
1282
fou_fini(void)1283 static void __exit fou_fini(void)
1284 {
1285 ip_tunnel_encap_del_fou_ops();
1286 genl_unregister_family(&fou_nl_family);
1287 unregister_pernet_device(&fou_net_ops);
1288 }
1289
1290 module_init(fou_init);
1291 module_exit(fou_fini);
1292 MODULE_AUTHOR("Tom Herbert <therbert@google.com>");
1293 MODULE_LICENSE("GPL");
1294 MODULE_DESCRIPTION("Foo over UDP");
1295
