1046 	int zc = 0;  in tcp_sendmsg_locked()  local
1055 				zc = MSG_ZEROCOPY;  in tcp_sendmsg_locked()
1064 				zc = MSG_ZEROCOPY;  in tcp_sendmsg_locked()
1070 			zc = MSG_SPLICE_PAGES;  in tcp_sendmsg_locked()
1175 		if (zc == 0) {  in tcp_sendmsg_locked()
1220 		} else if (zc == MSG_ZEROCOPY)  {  in tcp_sendmsg_locked()
1241 		} else if (zc == MSG_SPLICE_PAGES) {  in tcp_sendmsg_locked()
1807 					  struct tcp_zerocopy_receive *zc,  in tcp_zerocopy_set_hint_for_skb()  argument
1815 	zc->recv_skip_hint = skb->len - offset;  in tcp_zerocopy_set_hint_for_skb()
1831 		zc->recv_skip_hint -= partial_frag_remainder;  in tcp_zerocopy_set_hint_for_skb()
1839 	mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);  in tcp_zerocopy_set_hint_for_skb()
1840 	zc->recv_skip_hint = mappable_offset + partial_frag_remainder;  in tcp_zerocopy_set_hint_for_skb()
1847 				    struct tcp_zerocopy_receive *zc, int inq,  in receive_fallback_to_copy()  argument
1850 	unsigned long copy_address = (unsigned long)zc->copybuf_address;  in receive_fallback_to_copy()
1855 	zc->length = 0;  in receive_fallback_to_copy()
1856 	zc->recv_skip_hint = 0;  in receive_fallback_to_copy()
1858 	if (copy_address != zc->copybuf_address)  in receive_fallback_to_copy()
1867 				 tss, &zc->msg_flags);  in receive_fallback_to_copy()
1871 	zc->copybuf_len = err;  in receive_fallback_to_copy()
1872 	if (likely(zc->copybuf_len)) {  in receive_fallback_to_copy()
1878 			tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);  in receive_fallback_to_copy()
1883 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,  in tcp_copy_straggler_data()  argument
1887 	unsigned long copy_address = (unsigned long)zc->copybuf_address;  in tcp_copy_straggler_data()
1892 	if (copy_address != zc->copybuf_address)  in tcp_copy_straggler_data()
1902 	zc->recv_skip_hint -= copylen;  in tcp_copy_straggler_data()
1908 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,  in tcp_zc_handle_leftover()  argument
1915 	u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);  in tcp_zc_handle_leftover()
1926 			zc->msg_flags |= TCP_CMSG_TS;  in tcp_zc_handle_leftover()
1930 	zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,  in tcp_zc_handle_leftover()
1932 	return zc->copybuf_len < 0 ? 0 : copylen;  in tcp_zc_handle_leftover()
1941 					      struct tcp_zerocopy_receive *zc,  in tcp_zerocopy_vm_insert_batch_error()  argument
1947 	    zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {  in tcp_zerocopy_vm_insert_batch_error()
1978 		zc->recv_skip_hint += bytes_not_mapped;  in tcp_zerocopy_vm_insert_batch_error()
1989 					struct tcp_zerocopy_receive *zc,  in tcp_zerocopy_vm_insert_batch()  argument
2011 		pages_remaining, address, length, seq, zc, total_bytes_to_map,  in tcp_zerocopy_vm_insert_batch()
2017 				      struct tcp_zerocopy_receive *zc,  in tcp_zc_finalize_rx_tstamp()  argument
2023 	msg_control_addr = (unsigned long)zc->msg_control;  in tcp_zc_finalize_rx_tstamp()
2026 		(__kernel_size_t)zc->msg_controllen;  in tcp_zc_finalize_rx_tstamp()
2030 	zc->msg_flags = 0;  in tcp_zc_finalize_rx_tstamp()
2031 	if (zc->msg_control == msg_control_addr &&  in tcp_zc_finalize_rx_tstamp()
2032 	    zc->msg_controllen == cmsg_dummy.msg_controllen) {  in tcp_zc_finalize_rx_tstamp()
2034 		zc->msg_control = (__u64)  in tcp_zc_finalize_rx_tstamp()
2036 		zc->msg_controllen =  in tcp_zc_finalize_rx_tstamp()
2038 		zc->msg_flags = (__u32)cmsg_dummy.msg_flags;  in tcp_zc_finalize_rx_tstamp()
2069 				struct tcp_zerocopy_receive *zc,  in tcp_zerocopy_receive()  argument
2073 	unsigned long address = (unsigned long)zc->address;  in tcp_zerocopy_receive()
2075 	s32 copybuf_len = zc->copybuf_len;  in tcp_zerocopy_receive()
2087 	zc->copybuf_len = 0;  in tcp_zerocopy_receive()
2088 	zc->msg_flags = 0;  in tcp_zerocopy_receive()
2090 	if (address & (PAGE_SIZE - 1) || address != zc->address)  in tcp_zerocopy_receive()
2099 		return receive_fallback_to_copy(sk, zc, inq, tss);  in tcp_zerocopy_receive()
2102 		zc->length = 0;  in tcp_zerocopy_receive()
2103 		zc->recv_skip_hint = inq;  in tcp_zerocopy_receive()
2113 	vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);  in tcp_zerocopy_receive()
2117 		if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))  in tcp_zerocopy_receive()
2120 		zc->length = total_bytes_to_map;  in tcp_zerocopy_receive()
2121 		zc->recv_skip_hint = 0;  in tcp_zerocopy_receive()
2123 		zc->length = avail_len;  in tcp_zerocopy_receive()
2124 		zc->recv_skip_hint = avail_len;  in tcp_zerocopy_receive()
2127 	while (length + PAGE_SIZE <= zc->length) {  in tcp_zerocopy_receive()
2131 		if (zc->recv_skip_hint < PAGE_SIZE) {  in tcp_zerocopy_receive()
2135 				if (zc->recv_skip_hint > 0)  in tcp_zerocopy_receive()
2145 				zc->msg_flags |= TCP_CMSG_TS;  in tcp_zerocopy_receive()
2147 			zc->recv_skip_hint = skb->len - offset;  in tcp_zerocopy_receive()
2154 							  zc->recv_skip_hint);  in tcp_zerocopy_receive()
2156 			zc->recv_skip_hint = mappable_offset;  in tcp_zerocopy_receive()
2163 		zc->recv_skip_hint -= PAGE_SIZE;  in tcp_zerocopy_receive()
2166 		    zc->recv_skip_hint < PAGE_SIZE) {  in tcp_zerocopy_receive()
2173 							   &seq, zc,  in tcp_zerocopy_receive()
2183 						   zc, total_bytes_to_map);  in tcp_zerocopy_receive()
2192 		copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);  in tcp_zerocopy_receive()
2202 		if (length == zc->length)  in tcp_zerocopy_receive()
2203 			zc->recv_skip_hint = 0;  in tcp_zerocopy_receive()
2205 		if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))  in tcp_zerocopy_receive()
2208 	zc->length = length;  in tcp_zerocopy_receive()
4188 		struct tcp_zerocopy_receive zc = {};  in do_tcp_getsockopt()  local
4196 		if (unlikely(len > sizeof(zc))) {  in do_tcp_getsockopt()
4197 			err = check_zeroed_sockptr(optval, sizeof(zc),  in do_tcp_getsockopt()
4198 						   len - sizeof(zc));  in do_tcp_getsockopt()
4201 			len = sizeof(zc);  in do_tcp_getsockopt()
4205 		if (copy_from_sockptr(&zc, optval, len))  in do_tcp_getsockopt()
4207 		if (zc.reserved)  in do_tcp_getsockopt()
4209 		if (zc.msg_flags &  ~(TCP_VALID_ZC_MSG_FLAGS))  in do_tcp_getsockopt()
4212 		err = tcp_zerocopy_receive(sk, &zc, &tss);  in do_tcp_getsockopt()
4214 							  &zc, &len, err);  in do_tcp_getsockopt()
4235 		if (zc.msg_flags & TCP_CMSG_TS)  in do_tcp_getsockopt()
4236 			tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);  in do_tcp_getsockopt()
4238 			zc.msg_flags = 0;  in do_tcp_getsockopt()
4241 			zc.err = sock_error(sk);  in do_tcp_getsockopt()
4243 		zc.inq = tcp_inq_hint(sk);  in do_tcp_getsockopt()
4245 		if (!err && copy_to_sockptr(optval, &zc, len))  in do_tcp_getsockopt()