14 static void tcp_gso_tstamp(struct sk_buff *skb, unsigned int ts_seq,  in tcp_gso_tstamp()  argument
17 	while (skb) {  in tcp_gso_tstamp()
19 			skb_shinfo(skb)->tx_flags |= SKBTX_SW_TSTAMP;  in tcp_gso_tstamp()
20 			skb_shinfo(skb)->tskey = ts_seq;  in tcp_gso_tstamp()
24 		skb = skb->next;  in tcp_gso_tstamp()
29 static struct sk_buff *tcp4_gso_segment(struct sk_buff *skb,  in tcp4_gso_segment()  argument
32 	if (!(skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4))  in tcp4_gso_segment()
35 	if (!pskb_may_pull(skb, sizeof(struct tcphdr)))  in tcp4_gso_segment()
38 	if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {  in tcp4_gso_segment()
39 		const struct iphdr *iph = ip_hdr(skb);  in tcp4_gso_segment()
40 		struct tcphdr *th = tcp_hdr(skb);  in tcp4_gso_segment()
47 		skb->ip_summed = CHECKSUM_PARTIAL;  in tcp4_gso_segment()
48 		__tcp_v4_send_check(skb, iph->saddr, iph->daddr);  in tcp4_gso_segment()
51 	return tcp_gso_segment(skb, features);  in tcp4_gso_segment()
54 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,  in tcp_gso_segment()  argument
65 	struct sk_buff *gso_skb = skb;  in tcp_gso_segment()
69 	th = tcp_hdr(skb);  in tcp_gso_segment()
74 	if (!pskb_may_pull(skb, thlen))  in tcp_gso_segment()
77 	oldlen = (u16)~skb->len;  in tcp_gso_segment()
78 	__skb_pull(skb, thlen);  in tcp_gso_segment()
80 	mss = skb_shinfo(skb)->gso_size;  in tcp_gso_segment()
81 	if (unlikely(skb->len <= mss))  in tcp_gso_segment()
84 	if (skb_gso_ok(skb, features | NETIF_F_GSO_ROBUST)) {  in tcp_gso_segment()
87 		skb_shinfo(skb)->gso_segs = DIV_ROUND_UP(skb->len, mss);  in tcp_gso_segment()
96 	skb->ooo_okay = 0;  in tcp_gso_segment()
98 	segs = skb_segment(skb, features);  in tcp_gso_segment()
114 	skb = segs;  in tcp_gso_segment()
115 	th = tcp_hdr(skb);  in tcp_gso_segment()
124 	while (skb->next) {  in tcp_gso_segment()
128 		if (skb->ip_summed == CHECKSUM_PARTIAL)  in tcp_gso_segment()
129 			gso_reset_checksum(skb, ~th->check);  in tcp_gso_segment()
131 			th->check = gso_make_checksum(skb, ~th->check);  in tcp_gso_segment()
135 			skb->destructor = gso_skb->destructor;  in tcp_gso_segment()
136 			skb->sk = gso_skb->sk;  in tcp_gso_segment()
137 			sum_truesize += skb->truesize;  in tcp_gso_segment()
139 		skb = skb->next;  in tcp_gso_segment()
140 		th = tcp_hdr(skb);  in tcp_gso_segment()
154 		swap(gso_skb->sk, skb->sk);  in tcp_gso_segment()
155 		swap(gso_skb->destructor, skb->destructor);  in tcp_gso_segment()
156 		sum_truesize += skb->truesize;  in tcp_gso_segment()
162 			refcount_add(delta, &skb->sk->sk_wmem_alloc);  in tcp_gso_segment()
164 			WARN_ON_ONCE(refcount_sub_and_test(-delta, &skb->sk->sk_wmem_alloc));  in tcp_gso_segment()
167 	delta = htonl(oldlen + (skb_tail_pointer(skb) -  in tcp_gso_segment()
168 				skb_transport_header(skb)) +  in tcp_gso_segment()
169 		      skb->data_len);  in tcp_gso_segment()
172 	if (skb->ip_summed == CHECKSUM_PARTIAL)  in tcp_gso_segment()
173 		gso_reset_checksum(skb, ~th->check);  in tcp_gso_segment()
175 		th->check = gso_make_checksum(skb, ~th->check);  in tcp_gso_segment()
180 struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)  in tcp_gro_receive()  argument
195 	off = skb_gro_offset(skb);  in tcp_gro_receive()
197 	th = skb_gro_header_fast(skb, off);  in tcp_gro_receive()
198 	if (skb_gro_header_hard(skb, hlen)) {  in tcp_gro_receive()
199 		th = skb_gro_header_slow(skb, hlen, off);  in tcp_gro_receive()
209 	if (skb_gro_header_hard(skb, hlen)) {  in tcp_gro_receive()
210 		th = skb_gro_header_slow(skb, hlen, off);  in tcp_gro_receive()
215 	skb_gro_pull(skb, thlen);  in tcp_gro_receive()
217 	len = skb_gro_len(skb);  in tcp_gro_receive()
263 	flush |= p->decrypted ^ skb->decrypted;  in tcp_gro_receive()
266 	if (flush || skb_gro_receive(p, skb)) {  in tcp_gro_receive()
279 	if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))  in tcp_gro_receive()
283 	NAPI_GRO_CB(skb)->flush |= (flush != 0);  in tcp_gro_receive()
288 int tcp_gro_complete(struct sk_buff *skb)  in tcp_gro_complete()  argument
290 	struct tcphdr *th = tcp_hdr(skb);  in tcp_gro_complete()
292 	skb->csum_start = (unsigned char *)th - skb->head;  in tcp_gro_complete()
293 	skb->csum_offset = offsetof(struct tcphdr, check);  in tcp_gro_complete()
294 	skb->ip_summed = CHECKSUM_PARTIAL;  in tcp_gro_complete()
296 	skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;  in tcp_gro_complete()
299 		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;  in tcp_gro_complete()
306 struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)  in tcp4_gro_receive()  argument
309 	if (!NAPI_GRO_CB(skb)->flush &&  in tcp4_gro_receive()
310 	    skb_gro_checksum_validate(skb, IPPROTO_TCP,  in tcp4_gro_receive()
312 		NAPI_GRO_CB(skb)->flush = 1;  in tcp4_gro_receive()
316 	return tcp_gro_receive(head, skb);  in tcp4_gro_receive()
319 INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff)  in tcp4_gro_complete()  argument
321 	const struct iphdr *iph = ip_hdr(skb);  in tcp4_gro_complete()
322 	struct tcphdr *th = tcp_hdr(skb);  in tcp4_gro_complete()
324 	th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr,  in tcp4_gro_complete()
326 	skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;  in tcp4_gro_complete()
328 	if (NAPI_GRO_CB(skb)->is_atomic)  in tcp4_gro_complete()
329 		skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID;  in tcp4_gro_complete()
331 	return tcp_gro_complete(skb);  in tcp4_gro_complete()