1 /* SPDX-License-Identifier: GPL-2.0-or-later */
14 	/* Virtual address of skb_shinfo(skb)->frags[0].page + offset. */
20 	/* This indicates where we are processing relative to skb->data. */
23 	/* This is non-zero if the packet cannot be merged with the new skb. */
32 	/* Used in ipv6_gro_receive() and foo-over-udp */
47 		/* This is non-zero if the packet may be of the same flow. */
62 		/* Used in foo-over-udp, set in udp[46]_gro_receive */
85 #define NAPI_GRO_CB(skb) ((struct napi_gro_cb *)(skb)->cb)
90 	return ++NAPI_GRO_CB(skb)->recursion_counter == GRO_RECURSION_LIMIT;  in gro_recursion_inc_test()
99 		NAPI_GRO_CB(skb)->flush |= 1;
114 		NAPI_GRO_CB(skb)->flush |= 1;
123 	return NAPI_GRO_CB(skb)->data_offset;
128 	return skb->len - NAPI_GRO_CB(skb)->data_offset;  in skb_gro_len()
133 	NAPI_GRO_CB(skb)->data_offset += len;  in skb_gro_pull()
139 	return NAPI_GRO_CB(skb)->frag0 + offset;  in skb_gro_header_fast()
144 	return NAPI_GRO_CB(skb)->frag0_len < hlen;  in skb_gro_header_hard()
149 	NAPI_GRO_CB(skb)->frag0 = NULL;  in skb_gro_frag0_invalidate()
150 	NAPI_GRO_CB(skb)->frag0_len = 0;  in skb_gro_frag0_invalidate()
160 	return skb->data + offset;  in skb_gro_header_slow()
176 	return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) +  in skb_gro_network_header()
184 	return csum_tcpudp_nofold(iph->saddr, iph->daddr,  in inet_gro_compute_pseudo()
191 	if (NAPI_GRO_CB(skb)->csum_valid)  in skb_gro_postpull_rcsum()
192 		NAPI_GRO_CB(skb)->csum = wsum_negate(csum_partial(start, len,  in skb_gro_postpull_rcsum()
193 						wsum_negate(NAPI_GRO_CB(skb)->csum)));  in skb_gro_postpull_rcsum()
205 	return (NAPI_GRO_CB(skb)->gro_remcsum_start == skb_gro_offset(skb));  in skb_at_gro_remcsum_start()
212 	return ((skb->ip_summed != CHECKSUM_PARTIAL ||  in __skb_gro_checksum_validate_needed()
216 		NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in __skb_gro_checksum_validate_needed()
223 	if (NAPI_GRO_CB(skb)->csum_valid &&  in __skb_gro_checksum_validate_complete()
224 	    !csum_fold(csum_add(psum, NAPI_GRO_CB(skb)->csum)))  in __skb_gro_checksum_validate_complete()
227 	NAPI_GRO_CB(skb)->csum = psum;  in __skb_gro_checksum_validate_complete()
234 	if (NAPI_GRO_CB(skb)->csum_cnt > 0) {  in skb_gro_incr_csum_unnecessary()
236 		NAPI_GRO_CB(skb)->csum_cnt--;  in skb_gro_incr_csum_unnecessary()
270 	return (NAPI_GRO_CB(skb)->csum_cnt == 0 &&  in __skb_gro_checksum_convert_check()
271 		!NAPI_GRO_CB(skb)->csum_valid);  in __skb_gro_checksum_convert_check()
277 	NAPI_GRO_CB(skb)->csum = ~pseudo;  in __skb_gro_checksum_convert()
278 	NAPI_GRO_CB(skb)->csum_valid = 1;  in __skb_gro_checksum_convert()
295 	grc->offset = 0;  in skb_gro_remcsum_init()
296 	grc->delta = 0;  in skb_gro_remcsum_init()
308 	BUG_ON(!NAPI_GRO_CB(skb)->csum_valid);  in skb_gro_remcsum_process()
311 		NAPI_GRO_CB(skb)->gro_remcsum_start = off + hdrlen + start;  in skb_gro_remcsum_process()
319 	delta = remcsum_adjust(ptr + hdrlen, NAPI_GRO_CB(skb)->csum,  in skb_gro_remcsum_process()
322 	/* Adjust skb->csum since we changed the packet */  in skb_gro_remcsum_process()
323 	NAPI_GRO_CB(skb)->csum = csum_add(NAPI_GRO_CB(skb)->csum, delta);  in skb_gro_remcsum_process()
325 	grc->offset = off + hdrlen + offset;  in skb_gro_remcsum_process()
326 	grc->delta = delta;  in skb_gro_remcsum_process()
335 	size_t plen = grc->offset + sizeof(u16);  in skb_gro_remcsum_cleanup()
337 	if (!grc->delta)  in skb_gro_remcsum_cleanup()
340 	ptr = skb_gro_header(skb, plen, grc->offset);  in skb_gro_remcsum_cleanup()
344 	remcsum_unadjust((__sum16 *)ptr, grc->delta);  in skb_gro_remcsum_cleanup()
350 	if (PTR_ERR(pp) != -EINPROGRESS)  in skb_gro_flush_final()
351 		NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final()
358 	if (PTR_ERR(pp) != -EINPROGRESS) {  in skb_gro_flush_final_remcsum()
359 		NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final_remcsum()
361 		skb->remcsum_offload = 0;  in skb_gro_flush_final_remcsum()
367 	NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final()
374 	NAPI_GRO_CB(skb)->flush |= flush;  in skb_gro_flush_final_remcsum()
376 	skb->remcsum_offload = 0;  in skb_gro_flush_final_remcsum()
398 		NAPI_GRO_CB(skb)->flush |= 1, NULL :		\
422 	return ~csum_unfold(csum_ipv6_magic(&iph->saddr, &iph->daddr,  in ip6_gro_compute_pseudo()
431 	if (!napi->rx_count)  in gro_normal_list()
433 	netif_receive_skb_list_internal(&napi->rx_list);  in gro_normal_list()
434 	INIT_LIST_HEAD(&napi->rx_list);  in gro_normal_list()
435 	napi->rx_count = 0;  in gro_normal_list()
438 /* Queue one GRO_NORMAL SKB up for list processing. If batch size exceeded,
443 	list_add_tail(&skb->list, &napi->rx_list);  in gro_normal_one()
444 	napi->rx_count += segs;  in gro_normal_one()
445 	if (napi->rx_count >= READ_ONCE(gro_normal_batch))  in gro_normal_one()