15 static void tcp_gso_tstamp(struct sk_buff *skb, unsigned int ts_seq,  in tcp_gso_tstamp()  argument
18 	while (skb) {  in tcp_gso_tstamp()
20 			skb_shinfo(skb)->tx_flags |= SKBTX_SW_TSTAMP;  in tcp_gso_tstamp()
21 			skb_shinfo(skb)->tskey = ts_seq;  in tcp_gso_tstamp()
25 		skb = skb->next;  in tcp_gso_tstamp()
30 static struct sk_buff *tcp4_gso_segment(struct sk_buff *skb,  in tcp4_gso_segment()  argument
33 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4))  in tcp4_gso_segment()
36 	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))  in tcp4_gso_segment()
39 	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {  in tcp4_gso_segment()
40 		const struct iphdr *iph = ip_hdr(skb);  in tcp4_gso_segment()
41 		struct tcphdr *th = tcp_hdr(skb);  in tcp4_gso_segment()
48 		skb->ip_summed = CHECKSUM_PARTIAL;  in tcp4_gso_segment()
49 		__tcp_v4_send_check(skb, iph->saddr, iph->daddr);  in tcp4_gso_segment()
52 	return tcp_gso_segment(skb, features);  in tcp4_gso_segment()
55 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,  in tcp_gso_segment()  argument
66 	struct sk_buff *gso_skb = skb;  in tcp_gso_segment()
70 	th = tcp_hdr(skb);  in tcp_gso_segment()
75 	if (!pskb_may_pull(skb, thlen))  in tcp_gso_segment()
78 	oldlen = (u16)~skb->len;  in tcp_gso_segment()
79 	__skb_pull(skb, thlen);  in tcp_gso_segment()
81 	mss = skb_shinfo(skb)->gso_size;  in tcp_gso_segment()
82 	if (unlikely(skb->len <= mss))  in tcp_gso_segment()
85 	if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {  in tcp_gso_segment()
88 		skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);  in tcp_gso_segment()
97 	skb->ooo_okay = 0;  in tcp_gso_segment()
99 	segs = skb_segment(skb, features);  in tcp_gso_segment()
115 	skb = segs;  in tcp_gso_segment()
116 	th = tcp_hdr(skb);  in tcp_gso_segment()
125 	while (skb->next) {  in tcp_gso_segment()
129 		if (skb->ip_summed == CHECKSUM_PARTIAL)  in tcp_gso_segment()
130 			gso_reset_checksum(skb, ~th->check);  in tcp_gso_segment()
132 			th->check = gso_make_checksum(skb, ~th->check);  in tcp_gso_segment()
136 			skb->destructor = gso_skb->destructor;  in tcp_gso_segment()
137 			skb->sk = gso_skb->sk;  in tcp_gso_segment()
138 			sum_truesize += skb->truesize;  in tcp_gso_segment()
140 		skb = skb->next;  in tcp_gso_segment()
141 		th = tcp_hdr(skb);  in tcp_gso_segment()
155 		swap(gso_skb->sk, skb->sk);  in tcp_gso_segment()
156 		swap(gso_skb->destructor, skb->destructor);  in tcp_gso_segment()
157 		sum_truesize += skb->truesize;  in tcp_gso_segment()
163 			refcount_add(delta, &skb->sk->sk_wmem_alloc);  in tcp_gso_segment()
165 			WARN_ON_ONCE(refcount_sub_and_test(-delta, &skb->sk->sk_wmem_alloc));  in tcp_gso_segment()
168 	delta = htonl(oldlen + (skb_tail_pointer(skb) -  in tcp_gso_segment()
169 				skb_transport_header(skb)) +  in tcp_gso_segment()
170 		      skb->data_len);  in tcp_gso_segment()
173 	if (skb->ip_summed == CHECKSUM_PARTIAL)  in tcp_gso_segment()
174 		gso_reset_checksum(skb, ~th->check);  in tcp_gso_segment()
176 		th->check = gso_make_checksum(skb, ~th->check);  in tcp_gso_segment()
181 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)  in tcp_gro_receive()  argument
196 	off = skb_gro_offset(skb);  in tcp_gro_receive()
198 	th = skb_gro_header(skb, hlen, off);  in tcp_gro_receive()
207 	if (skb_gro_header_hard(skb, hlen)) {  in tcp_gro_receive()
208 		th = skb_gro_header_slow(skb, hlen, off);  in tcp_gro_receive()
213 	skb_gro_pull(skb, thlen);  in tcp_gro_receive()
215 	len = skb_gro_len(skb);  in tcp_gro_receive()
262 	if (unlikely(skb_is_gso(skb)))  in tcp_gro_receive()
263 		flush |= (mss != skb_shinfo(skb)->gso_size);  in tcp_gro_receive()
269 	flush |= p->decrypted ^ skb->decrypted;  in tcp_gro_receive()
272 	if (flush || skb_gro_receive(p, skb)) {  in tcp_gro_receive()
281 	if (unlikely(skb_is_gso(skb)))  in tcp_gro_receive()
282 		flush = len != NAPI_GRO_CB(skb)->count * skb_shinfo(skb)->gso_size;  in tcp_gro_receive()
290 	if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))  in tcp_gro_receive()
294 	NAPI_GRO_CB(skb)->flush |= (flush != 0);  in tcp_gro_receive()
299 int tcp_gro_complete(struct sk_buff *skb)  in tcp_gro_complete()  argument
301 	struct tcphdr *th = tcp_hdr(skb);  in tcp_gro_complete()
303 	skb->csum_start = (unsigned char *)th - skb->head;  in tcp_gro_complete()
304 	skb->csum_offset = offsetof(struct tcphdr, check);  in tcp_gro_complete()
305 	skb->ip_summed = CHECKSUM_PARTIAL;  in tcp_gro_complete()
307 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in tcp_gro_complete()
310 		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;  in tcp_gro_complete()
312 	if (skb->encapsulation)  in tcp_gro_complete()
313 		skb->inner_transport_header = skb->transport_header;  in tcp_gro_complete()
320 struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in tcp4_gro_receive()  argument
323 	if (!NAPI_GRO_CB(skb)->flush &&  in tcp4_gro_receive()
324 	    skb_gro_checksum_validate(skb, IPPROTO_TCP,  in tcp4_gro_receive()
326 		NAPI_GRO_CB(skb)->flush = 1;  in tcp4_gro_receive()
330 	return tcp_gro_receive(head, skb);  in tcp4_gro_receive()
333 INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff)  in tcp4_gro_complete()  argument
335 	const struct iphdr *iph = ip_hdr(skb);  in tcp4_gro_complete()
336 	struct tcphdr *th = tcp_hdr(skb);  in tcp4_gro_complete()
338 	th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr,  in tcp4_gro_complete()
340 	skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;  in tcp4_gro_complete()
342 	if (NAPI_GRO_CB(skb)->is_atomic)  in tcp4_gro_complete()
343 		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;  in tcp4_gro_complete()
345 	return tcp_gro_complete(skb);  in tcp4_gro_complete()