15 static struct sk_buff *__skb_udp_tunnel_segment(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
17 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in __skb_udp_tunnel_segment()  argument
21 	int tnl_hlen = skb_inner_mac_header(skb) - skb_transport_header(skb);  in __skb_udp_tunnel_segment()
24 	struct udphdr *uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
25 	u16 mac_offset = skb->mac_header;  in __skb_udp_tunnel_segment()
26 	__be16 protocol = skb->protocol;  in __skb_udp_tunnel_segment()
27 	u16 mac_len = skb->mac_len;  in __skb_udp_tunnel_segment()
32 	if (unlikely(!pskb_may_pull(skb, tnl_hlen)))  in __skb_udp_tunnel_segment()
41 	if (skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL)  in __skb_udp_tunnel_segment()
44 		partial = (__force __wsum)htonl(skb->len);  in __skb_udp_tunnel_segment()
48 	skb->encapsulation = 0;  in __skb_udp_tunnel_segment()
49 	SKB_GSO_CB(skb)->encap_level = 0;  in __skb_udp_tunnel_segment()
50 	__skb_pull(skb, tnl_hlen);  in __skb_udp_tunnel_segment()
51 	skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
52 	skb_set_network_header(skb, skb_inner_network_offset(skb));  in __skb_udp_tunnel_segment()
53 	skb_set_transport_header(skb, skb_inner_transport_offset(skb));  in __skb_udp_tunnel_segment()
54 	skb->mac_len = skb_inner_network_offset(skb);  in __skb_udp_tunnel_segment()
55 	skb->protocol = new_protocol;  in __skb_udp_tunnel_segment()
57 	need_csum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM);  in __skb_udp_tunnel_segment()
58 	skb->encap_hdr_csum = need_csum;  in __skb_udp_tunnel_segment()
60 	remcsum = !!(skb_shinfo(skb)->gso_type & SKB_GSO_TUNNEL_REMCSUM);  in __skb_udp_tunnel_segment()
61 	skb->remcsum_offload = remcsum;  in __skb_udp_tunnel_segment()
63 	need_ipsec = skb_dst(skb) && dst_xfrm(skb_dst(skb));  in __skb_udp_tunnel_segment()
67 			  (skb->dev->features &  in __skb_udp_tunnel_segment()
71 	features &= skb->dev->hw_enc_features;  in __skb_udp_tunnel_segment()
86 	segs = gso_inner_segment(skb, features);  in __skb_udp_tunnel_segment()
88 		skb_gso_error_unwind(skb, protocol, tnl_hlen, mac_offset,  in __skb_udp_tunnel_segment()
95 	outer_hlen = skb_tnl_header_len(skb);  in __skb_udp_tunnel_segment()
97 	skb = segs;  in __skb_udp_tunnel_segment()
102 			skb->ip_summed = CHECKSUM_NONE;  in __skb_udp_tunnel_segment()
105 		if (skb->ip_summed == CHECKSUM_PARTIAL) {  in __skb_udp_tunnel_segment()
106 			skb_reset_inner_headers(skb);  in __skb_udp_tunnel_segment()
107 			skb->encapsulation = 1;  in __skb_udp_tunnel_segment()
110 		skb->mac_len = mac_len;  in __skb_udp_tunnel_segment()
111 		skb->protocol = protocol;  in __skb_udp_tunnel_segment()
113 		__skb_push(skb, outer_hlen);  in __skb_udp_tunnel_segment()
114 		skb_reset_mac_header(skb);  in __skb_udp_tunnel_segment()
115 		skb_set_network_header(skb, mac_len);  in __skb_udp_tunnel_segment()
116 		skb_set_transport_header(skb, udp_offset);  in __skb_udp_tunnel_segment()
117 		len = skb->len - udp_offset;  in __skb_udp_tunnel_segment()
118 		uh = udp_hdr(skb);  in __skb_udp_tunnel_segment()
124 		if (gso_partial && skb_is_gso(skb)) {  in __skb_udp_tunnel_segment()
125 			uh->len = htons(skb_shinfo(skb)->gso_size +  in __skb_udp_tunnel_segment()
126 					SKB_GSO_CB(skb)->data_offset +  in __skb_udp_tunnel_segment()
127 					skb->head - (unsigned char *)uh);  in __skb_udp_tunnel_segment()
138 		if (skb->encapsulation || !offload_csum) {  in __skb_udp_tunnel_segment()
139 			uh->check = gso_make_checksum(skb, ~uh->check);  in __skb_udp_tunnel_segment()
143 			skb->ip_summed = CHECKSUM_PARTIAL;  in __skb_udp_tunnel_segment()
144 			skb->csum_start = skb_transport_header(skb) - skb->head;  in __skb_udp_tunnel_segment()
145 			skb->csum_offset = offsetof(struct udphdr, check);  in __skb_udp_tunnel_segment()
147 	} while ((skb = skb->next));  in __skb_udp_tunnel_segment()
152 struct sk_buff *skb_udp_tunnel_segment(struct sk_buff *skb,  in skb_udp_tunnel_segment()  argument
157 	__be16 protocol = skb->protocol;  in skb_udp_tunnel_segment()
160 	struct sk_buff *(*gso_inner_segment)(struct sk_buff *skb,  in skb_udp_tunnel_segment()
165 	switch (skb->inner_protocol_type) {  in skb_udp_tunnel_segment()
167 		protocol = skb->inner_protocol;  in skb_udp_tunnel_segment()
172 		ops = rcu_dereference(offloads[skb->inner_ipproto]);  in skb_udp_tunnel_segment()
181 	segs = __skb_udp_tunnel_segment(skb, features, gso_inner_segment,  in skb_udp_tunnel_segment()
249 static struct sk_buff *__udp_gso_segment_list(struct sk_buff *skb,  in __udp_gso_segment_list()  argument
253 	unsigned int mss = skb_shinfo(skb)->gso_size;  in __udp_gso_segment_list()
255 	skb = skb_segment_list(skb, features, skb_mac_header_len(skb));  in __udp_gso_segment_list()
256 	if (IS_ERR(skb))  in __udp_gso_segment_list()
257 		return skb;  in __udp_gso_segment_list()
259 	udp_hdr(skb)->len = htons(sizeof(struct udphdr) + mss);  in __udp_gso_segment_list()
261 	return is_ipv6 ? skb : __udpv4_gso_segment_list_csum(skb);  in __udp_gso_segment_list()
368 static struct sk_buff *udp4_ufo_fragment(struct sk_buff *skb,  in udp4_ufo_fragment()  argument
377 	if (skb->encapsulation &&  in udp4_ufo_fragment()
378 	    (skb_shinfo(skb)->gso_type &  in udp4_ufo_fragment()
380 		segs = skb_udp_tunnel_segment(skb, features, false);  in udp4_ufo_fragment()
384 	if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))  in udp4_ufo_fragment()
387 	if (!pskb_may_pull(skb, sizeof(struct udphdr)))  in udp4_ufo_fragment()
390 	if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)  in udp4_ufo_fragment()
391 		return __udp_gso_segment(skb, features, false);  in udp4_ufo_fragment()
393 	mss = skb_shinfo(skb)->gso_size;  in udp4_ufo_fragment()
394 	if (unlikely(skb->len <= mss))  in udp4_ufo_fragment()
402 	uh = udp_hdr(skb);  in udp4_ufo_fragment()
403 	iph = ip_hdr(skb);  in udp4_ufo_fragment()
406 	csum = skb_checksum(skb, 0, skb->len, 0);  in udp4_ufo_fragment()
407 	uh->check = udp_v4_check(skb->len, iph->saddr, iph->daddr, csum);  in udp4_ufo_fragment()
411 	skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_ufo_fragment()
417 	if (!skb->encap_hdr_csum)  in udp4_ufo_fragment()
423 	segs = skb_segment(skb, features);  in udp4_ufo_fragment()
428 static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)  in skb_gro_receive_list()  argument
430 	if (unlikely(p->len + skb->len >= 65536))  in skb_gro_receive_list()
434 		skb_shinfo(p)->frag_list = skb;  in skb_gro_receive_list()
436 		NAPI_GRO_CB(p)->last->next = skb;  in skb_gro_receive_list()
438 	skb_pull(skb, skb_gro_offset(skb));  in skb_gro_receive_list()
440 	NAPI_GRO_CB(p)->last = skb;  in skb_gro_receive_list()
442 	p->data_len += skb->len;  in skb_gro_receive_list()
445 	skb->destructor = NULL;  in skb_gro_receive_list()
446 	p->truesize += skb->truesize;  in skb_gro_receive_list()
447 	p->len += skb->len;  in skb_gro_receive_list()
449 	NAPI_GRO_CB(skb)->same_flow = 1;  in skb_gro_receive_list()
457 					       struct sk_buff *skb)  in udp_gro_receive_segment()  argument
459 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp_gro_receive_segment()
468 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
474 	if (ulen <= sizeof(*uh) || ulen != skb_gro_len(skb)) {  in udp_gro_receive_segment()
475 		NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
479 	skb_gro_pull(skb, sizeof(struct udphdr));  in udp_gro_receive_segment()
493 		if (NAPI_GRO_CB(skb)->is_flist != NAPI_GRO_CB(p)->is_flist) {  in udp_gro_receive_segment()
494 			NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
507 			if (NAPI_GRO_CB(skb)->is_flist) {  in udp_gro_receive_segment()
508 				if (!pskb_may_pull(skb, skb_gro_offset(skb))) {  in udp_gro_receive_segment()
509 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
512 				if ((skb->ip_summed != p->ip_summed) ||  in udp_gro_receive_segment()
513 				    (skb->csum_level != p->csum_level)) {  in udp_gro_receive_segment()
514 					NAPI_GRO_CB(skb)->flush = 1;  in udp_gro_receive_segment()
517 				ret = skb_gro_receive_list(p, skb);  in udp_gro_receive_segment()
519 				skb_gro_postpull_rcsum(skb, uh,  in udp_gro_receive_segment()
522 				ret = skb_gro_receive(p, skb);  in udp_gro_receive_segment()
537 struct sk_buff *udp_gro_receive(struct list_head *head, struct sk_buff *skb,  in udp_gro_receive()  argument
543 	unsigned int off = skb_gro_offset(skb);  in udp_gro_receive()
549 	NAPI_GRO_CB(skb)->is_flist = 0;  in udp_gro_receive()
551 		if (skb->dev->features & NETIF_F_GRO_FRAGLIST)  in udp_gro_receive()
552 			NAPI_GRO_CB(skb)->is_flist = sk ? !udp_sk(sk)->gro_enabled : 1;  in udp_gro_receive()
554 		if ((!sk && (skb->dev->features & NETIF_F_GRO_UDP_FWD)) ||  in udp_gro_receive()
555 		    (sk && udp_sk(sk)->gro_enabled) || NAPI_GRO_CB(skb)->is_flist)  in udp_gro_receive()
556 			return call_gro_receive(udp_gro_receive_segment, head, skb);  in udp_gro_receive()
562 	if (NAPI_GRO_CB(skb)->encap_mark ||  in udp_gro_receive()
563 	    (uh->check && skb->ip_summed != CHECKSUM_PARTIAL &&  in udp_gro_receive()
564 	     NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in udp_gro_receive()
565 	     !NAPI_GRO_CB(skb)->csum_valid))  in udp_gro_receive()
569 	NAPI_GRO_CB(skb)->encap_mark = 1;  in udp_gro_receive()
589 	skb_gro_pull(skb, sizeof(struct udphdr)); /* pull encapsulating udp header */  in udp_gro_receive()
590 	skb_gro_postpull_rcsum(skb, uh, sizeof(struct udphdr));  in udp_gro_receive()
591 	pp = call_gro_receive_sk(udp_sk(sk)->gro_receive, sk, head, skb);  in udp_gro_receive()
594 	skb_gro_flush_final(skb, pp, flush);  in udp_gro_receive()
599 static struct sock *udp4_gro_lookup_skb(struct sk_buff *skb, __be16 sport,  in udp4_gro_lookup_skb()  argument
602 	const struct iphdr *iph = skb_gro_network_header(skb);  in udp4_gro_lookup_skb()
604 	return __udp4_lib_lookup(dev_net(skb->dev), iph->saddr, sport,  in udp4_gro_lookup_skb()
605 				 iph->daddr, dport, inet_iif(skb),  in udp4_gro_lookup_skb()
606 				 inet_sdif(skb), &udp_table, NULL);  in udp4_gro_lookup_skb()
610 struct sk_buff *udp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in udp4_gro_receive()  argument
612 	struct udphdr *uh = udp_gro_udphdr(skb);  in udp4_gro_receive()
620 	if (NAPI_GRO_CB(skb)->flush)  in udp4_gro_receive()
623 	if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,  in udp4_gro_receive()
627 		skb_gro_checksum_try_convert(skb, IPPROTO_UDP,  in udp4_gro_receive()
630 	NAPI_GRO_CB(skb)->is_ipv6 = 0;  in udp4_gro_receive()
633 		sk = udp4_gro_lookup_skb(skb, uh->source, uh->dest);  in udp4_gro_receive()
635 	pp = udp_gro_receive(head, skb, uh, sk);  in udp4_gro_receive()
639 	NAPI_GRO_CB(skb)->flush = 1;  in udp4_gro_receive()
643 static int udp_gro_complete_segment(struct sk_buff *skb)  in udp_gro_complete_segment()  argument
645 	struct udphdr *uh = udp_hdr(skb);  in udp_gro_complete_segment()
647 	skb->csum_start = (unsigned char *)uh - skb->head;  in udp_gro_complete_segment()
648 	skb->csum_offset = offsetof(struct udphdr, check);  in udp_gro_complete_segment()
649 	skb->ip_summed = CHECKSUM_PARTIAL;  in udp_gro_complete_segment()
651 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp_gro_complete_segment()
652 	skb_shinfo(skb)->gso_type |= SKB_GSO_UDP_L4;  in udp_gro_complete_segment()
654 	if (skb->encapsulation)  in udp_gro_complete_segment()
655 		skb->inner_transport_header = skb->transport_header;  in udp_gro_complete_segment()
660 int udp_gro_complete(struct sk_buff *skb, int nhoff,  in udp_gro_complete()  argument
663 	__be16 newlen = htons(skb->len - nhoff);  in udp_gro_complete()
664 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp_gro_complete()
671 				udp4_lib_lookup_skb, skb, uh->source, uh->dest);  in udp_gro_complete()
673 		skb_shinfo(skb)->gso_type = uh->check ? SKB_GSO_UDP_TUNNEL_CSUM  in udp_gro_complete()
679 		NAPI_GRO_CB(skb)->encap_mark = 0;  in udp_gro_complete()
684 		skb->encapsulation = 1;  in udp_gro_complete()
685 		err = udp_sk(sk)->gro_complete(sk, skb,  in udp_gro_complete()
688 		err = udp_gro_complete_segment(skb);  in udp_gro_complete()
691 	if (skb->remcsum_offload)  in udp_gro_complete()
692 		skb_shinfo(skb)->gso_type |= SKB_GSO_TUNNEL_REMCSUM;  in udp_gro_complete()
698 INDIRECT_CALLABLE_SCOPE int udp4_gro_complete(struct sk_buff *skb, int nhoff)  in udp4_gro_complete()  argument
700 	const struct iphdr *iph = ip_hdr(skb);  in udp4_gro_complete()
701 	struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);  in udp4_gro_complete()
704 	if (NAPI_GRO_CB(skb)->is_flist && !NAPI_GRO_CB(skb)->encap_mark) {  in udp4_gro_complete()
705 		uh->len = htons(skb->len - nhoff);  in udp4_gro_complete()
707 		skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);  in udp4_gro_complete()
708 		skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in udp4_gro_complete()
710 		if (skb->ip_summed == CHECKSUM_UNNECESSARY) {  in udp4_gro_complete()
711 			if (skb->csum_level < SKB_MAX_CSUM_LEVEL)  in udp4_gro_complete()
712 				skb->csum_level++;  in udp4_gro_complete()
714 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in udp4_gro_complete()
715 			skb->csum_level = 0;  in udp4_gro_complete()
722 		uh->check = ~udp_v4_check(skb->len - nhoff, iph->saddr,  in udp4_gro_complete()
725 	return udp_gro_complete(skb, nhoff, udp4_lib_lookup_skb);  in udp4_gro_complete()