32 #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb)	\  argument
34 	unlikely(gro_recursion_inc_test(skb)) ?			\
35 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
36 		INDIRECT_CALL_L4(cb, f2, f1, head, skb);	\
39 static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)  in ipv6_gso_pull_exthdrs()  argument
57 		if (unlikely(!pskb_may_pull(skb, 8)))  in ipv6_gso_pull_exthdrs()
60 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
63 		if (unlikely(!pskb_may_pull(skb, len)))  in ipv6_gso_pull_exthdrs()
66 		opth = (void *)skb->data;  in ipv6_gso_pull_exthdrs()
68 		__skb_pull(skb, len);  in ipv6_gso_pull_exthdrs()
74 static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,  in ipv6_gso_segment()  argument
89 	skb_reset_network_header(skb);  in ipv6_gso_segment()
90 	nexthdr = ipv6_has_hopopt_jumbo(skb);  in ipv6_gso_segment()
95 		err = skb_cow_head(skb, 0);  in ipv6_gso_segment()
102 		memmove(skb_mac_header(skb) + hophdr_len,  in ipv6_gso_segment()
103 			skb_mac_header(skb),  in ipv6_gso_segment()
105 		skb->data += hophdr_len;  in ipv6_gso_segment()
106 		skb->len -= hophdr_len;  in ipv6_gso_segment()
107 		skb->network_header += hophdr_len;  in ipv6_gso_segment()
108 		skb->mac_header += hophdr_len;  in ipv6_gso_segment()
109 		ipv6h = (struct ipv6hdr *)skb->data;  in ipv6_gso_segment()
112 	nhoff = skb_network_header(skb) - skb_mac_header(skb);  in ipv6_gso_segment()
113 	if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))  in ipv6_gso_segment()
116 	encap = SKB_GSO_CB(skb)->encap_level > 0;  in ipv6_gso_segment()
118 		features &= skb->dev->hw_enc_features;  in ipv6_gso_segment()
119 	SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);  in ipv6_gso_segment()
121 	ipv6h = ipv6_hdr(skb);  in ipv6_gso_segment()
122 	__skb_pull(skb, sizeof(*ipv6h));  in ipv6_gso_segment()
125 	proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);  in ipv6_gso_segment()
127 	if (skb->encapsulation &&  in ipv6_gso_segment()
128 	    skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))  in ipv6_gso_segment()
130 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
132 		udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&  in ipv6_gso_segment()
133 			  (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);  in ipv6_gso_segment()
137 		skb_reset_transport_header(skb);  in ipv6_gso_segment()
138 		segs = ops->callbacks.gso_segment(skb, features);  in ipv6_gso_segment()
140 			skb->network_header = skb_mac_header(skb) + nhoff - skb->head;  in ipv6_gso_segment()
148 	for (skb = segs; skb; skb = skb->next) {  in ipv6_gso_segment()
149 		ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);  in ipv6_gso_segment()
150 		if (gso_partial && skb_is_gso(skb))  in ipv6_gso_segment()
151 			payload_len = skb_shinfo(skb)->gso_size +  in ipv6_gso_segment()
152 				      SKB_GSO_CB(skb)->data_offset +  in ipv6_gso_segment()
153 				      skb->head - (unsigned char *)(ipv6h + 1);  in ipv6_gso_segment()
155 			payload_len = skb->len - nhoff - sizeof(*ipv6h);  in ipv6_gso_segment()
157 		skb->network_header = (u8 *)ipv6h - skb->head;  in ipv6_gso_segment()
158 		skb_reset_mac_len(skb);  in ipv6_gso_segment()
161 			int err = ip6_find_1stfragopt(skb, &prevhdr);  in ipv6_gso_segment()
168 			if (skb->next)  in ipv6_gso_segment()
174 			skb_reset_inner_headers(skb);  in ipv6_gso_segment()
208 							 struct sk_buff *skb)  in ipv6_gro_receive()  argument
220 	off = skb_gro_offset(skb);  in ipv6_gro_receive()
222 	iph = skb_gro_header(skb, hlen, off);  in ipv6_gro_receive()
226 	skb_set_network_header(skb, off);  in ipv6_gro_receive()
227 	skb_gro_pull(skb, sizeof(*iph));  in ipv6_gro_receive()
228 	skb_set_transport_header(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
230 	flush += ntohs(iph->payload_len) != skb_gro_len(skb);  in ipv6_gro_receive()
235 		pskb_pull(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
236 		skb_gro_frag0_invalidate(skb);  in ipv6_gro_receive()
237 		proto = ipv6_gso_pull_exthdrs(skb, proto);  in ipv6_gro_receive()
238 		skb_gro_pull(skb, -skb_transport_offset(skb));  in ipv6_gro_receive()
239 		skb_reset_transport_header(skb);  in ipv6_gro_receive()
240 		__skb_push(skb, skb_gro_offset(skb));  in ipv6_gro_receive()
246 		iph = ipv6_hdr(skb);  in ipv6_gro_receive()
249 	NAPI_GRO_CB(skb)->proto = proto;  in ipv6_gro_receive()
252 	nlen = skb_network_header_len(skb);  in ipv6_gro_receive()
291 		if (NAPI_GRO_CB(skb)->is_atomic)  in ipv6_gro_receive()
295 	NAPI_GRO_CB(skb)->is_atomic = true;  in ipv6_gro_receive()
296 	NAPI_GRO_CB(skb)->flush |= flush;  in ipv6_gro_receive()
298 	skb_gro_postpull_rcsum(skb, iph, nlen);  in ipv6_gro_receive()
301 					 ops->callbacks.gro_receive, head, skb);  in ipv6_gro_receive()
304 	skb_gro_flush_final(skb, pp, flush);  in ipv6_gro_receive()
310 					      struct sk_buff *skb)  in sit_ip6ip6_gro_receive()  argument
314 	if (NAPI_GRO_CB(skb)->encap_mark) {  in sit_ip6ip6_gro_receive()
315 		NAPI_GRO_CB(skb)->flush = 1;  in sit_ip6ip6_gro_receive()
319 	NAPI_GRO_CB(skb)->encap_mark = 1;  in sit_ip6ip6_gro_receive()
321 	return ipv6_gro_receive(head, skb);  in sit_ip6ip6_gro_receive()
325 					  struct sk_buff *skb)  in ip4ip6_gro_receive()  argument
329 	if (NAPI_GRO_CB(skb)->encap_mark) {  in ip4ip6_gro_receive()
330 		NAPI_GRO_CB(skb)->flush = 1;  in ip4ip6_gro_receive()
334 	NAPI_GRO_CB(skb)->encap_mark = 1;  in ip4ip6_gro_receive()
336 	return inet_gro_receive(head, skb);  in ip4ip6_gro_receive()
339 INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)  in ipv6_gro_complete()  argument
346 	if (skb->encapsulation) {  in ipv6_gro_complete()
347 		skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));  in ipv6_gro_complete()
348 		skb_set_inner_network_header(skb, nhoff);  in ipv6_gro_complete()
351 	payload_len = skb->len - nhoff - sizeof(*iph);  in ipv6_gro_complete()
357 		memmove(skb_mac_header(skb) - hoplen, skb_mac_header(skb),  in ipv6_gro_complete()
358 			skb->transport_header - skb->mac_header);  in ipv6_gro_complete()
359 		skb->data -= hoplen;  in ipv6_gro_complete()
360 		skb->len += hoplen;  in ipv6_gro_complete()
361 		skb->mac_header -= hoplen;  in ipv6_gro_complete()
362 		skb->network_header -= hoplen;  in ipv6_gro_complete()
363 		iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
376 		iph = (struct ipv6hdr *)(skb->data + nhoff);  in ipv6_gro_complete()
385 			       udp6_gro_complete, skb, nhoff);  in ipv6_gro_complete()
391 static int sit_gro_complete(struct sk_buff *skb, int nhoff)  in sit_gro_complete()  argument
393 	skb->encapsulation = 1;  in sit_gro_complete()
394 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;  in sit_gro_complete()
395 	return ipv6_gro_complete(skb, nhoff);  in sit_gro_complete()
398 static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip6ip6_gro_complete()  argument
400 	skb->encapsulation = 1;  in ip6ip6_gro_complete()
401 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip6ip6_gro_complete()
402 	return ipv6_gro_complete(skb, nhoff);  in ip6ip6_gro_complete()
405 static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)  in ip4ip6_gro_complete()  argument
407 	skb->encapsulation = 1;  in ip4ip6_gro_complete()
408 	skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;  in ip4ip6_gro_complete()
409 	return inet_gro_complete(skb, nhoff);  in ip4ip6_gro_complete()
421 static struct sk_buff *sit_gso_segment(struct sk_buff *skb,  in sit_gso_segment()  argument
424 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))  in sit_gso_segment()
427 	return ipv6_gso_segment(skb, features);  in sit_gso_segment()
430 static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,  in ip4ip6_gso_segment()  argument
433 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip4ip6_gso_segment()
436 	return inet_gso_segment(skb, features);  in ip4ip6_gso_segment()
439 static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,  in ip6ip6_gso_segment()  argument
442 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))  in ip6ip6_gso_segment()
445 	return ipv6_gso_segment(skb, features);  in ip6ip6_gso_segment()