1 /* SPDX-License-Identifier: GPL-2.0-or-later */
46 #include <linux/bpf-cgroup.h>
59 #define TCP_MIN_GSO_SIZE	(TCP_MIN_SND_MSS - MAX_TCP_OPTION_SPACE)
67 /* Minimal accepted MSS. It is (60+60+8) - (20+20). */
82 /* Maximal number of ACKs sent quickly to accelerate slow-start. */
97 				 * to ~3sec-8min depending on RTO.
104 				 * 15 is ~13-30min depending on RTO.
123 #define TCP_TIMEWAIT_LEN (60*HZ) /* how long to wait to destroy TIME-WAIT
128 				  * to combine FIN-WAIT-2 timeout with
129 				  * TIME-WAIT timer.
133 #define TCP_DELACK_MAX	((unsigned)(HZ/5))	/* maximal time to delay before sending an ACK */
135 #define TCP_DELACK_MIN	((unsigned)(HZ/25))	/* minimal time to delay before sending an ACK */
148 						 * valid RTT sample has been acquired,
167 #define TCP_PAWS_MSL	60		/* Per-host timestamps are invalidated
173 #define TCP_PAWS_WINDOW	1		/* Replay window for per-host
193  * experimental options. See draft-ietf-tcpm-experimental-options-00.txt
222 /* Flags in tp->nonagle */
227 /* TCP thin-stream limits */
264 	if (mem_cgroup_sockets_enabled && sk->sk_memcg &&  in tcp_under_memory_pressure()
265 	    mem_cgroup_under_socket_pressure(sk->sk_memcg))  in tcp_under_memory_pressure()
277         return (__s32)(seq1-seq2) < 0;  in before()
284 	return seq3 - seq2 >= seq1 - seq2;  in between()
289 	if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&  in tcp_out_of_memory()
297 	sk_wmem_queued_add(sk, -skb->truesize);  in tcp_wmem_free_skb()
299 		sk_mem_uncharge(sk, skb->truesize);  in tcp_wmem_free_skb()
312 #define TCP_INC_STATS(net, field)	SNMP_INC_STATS((net)->mib.tcp_statistics, field)
313 #define __TCP_INC_STATS(net, field)	__SNMP_INC_STATS((net)->mib.tcp_statistics, field)
314 #define TCP_DEC_STATS(net, field)	SNMP_DEC_STATS((net)->mib.tcp_statistics, field)
315 #define TCP_ADD_STATS(net, field, val)	SNMP_ADD_STATS((net)->mib.tcp_statistics, field, val)
364 	if (icsk->icsk_ack.quick) {  in tcp_dec_quickack_mode()
365 		if (pkts >= icsk->icsk_ack.quick) {  in tcp_dec_quickack_mode()
366 			icsk->icsk_ack.quick = 0;  in tcp_dec_quickack_mode()
368 			icsk->icsk_ack.ato   = TCP_ATO_MIN;  in tcp_dec_quickack_mode()
370 			icsk->icsk_ack.quick -= pkts;  in tcp_dec_quickack_mode()
438  *	BPF SKB-less helpers
517 	if (sk->sk_reuseport) {  in tcp_synq_overflow()
520 		reuse = rcu_dereference(sk->sk_reuseport_cb);  in tcp_synq_overflow()
522 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);  in tcp_synq_overflow()
525 				WRITE_ONCE(reuse->synq_overflow_ts, now);  in tcp_synq_overflow()
530 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);  in tcp_synq_overflow()
532 		WRITE_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp, now);  in tcp_synq_overflow()
541 	if (sk->sk_reuseport) {  in tcp_synq_no_recent_overflow()
544 		reuse = rcu_dereference(sk->sk_reuseport_cb);  in tcp_synq_no_recent_overflow()
546 			last_overflow = READ_ONCE(reuse->synq_overflow_ts);  in tcp_synq_no_recent_overflow()
547 			return !time_between32(now, last_overflow - HZ,  in tcp_synq_no_recent_overflow()
553 	last_overflow = READ_ONCE(tcp_sk(sk)->rx_opt.ts_recent_stamp);  in tcp_synq_no_recent_overflow()
557 	 * 'last_overflow - HZ' as lower bound. That's because a concurrent  in tcp_synq_no_recent_overflow()
562 	return !time_between32(now, last_overflow - HZ,  in tcp_synq_no_recent_overflow()
640 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->pacing_timer) == 1)  in tcp_clear_xmit_timers()
643 	if (hrtimer_try_to_cancel(&tcp_sk(sk)->compressed_ack_timer) == 1)  in tcp_clear_xmit_timers()
659 	 * to sub-MSS pieces for the sake of SWS or making sure there  in tcp_bound_to_half_wnd()
665 	if (tp->max_window > TCP_MSS_DEFAULT)  in tcp_bound_to_half_wnd()
666 		cutoff = (tp->max_window >> 1);  in tcp_bound_to_half_wnd()
668 		cutoff = tp->max_window;  in tcp_bound_to_half_wnd()
671 		return max_t(int, cutoff, 68U - tp->tcp_header_len);  in tcp_bound_to_half_wnd()
679 /* Read 'sendfile()'-style from a TCP socket */
694 	if (inet_csk(sk)->icsk_rto > TCP_RTO_MAX)  in tcp_bound_rto()
695 		inet_csk(sk)->icsk_rto = TCP_RTO_MAX;  in tcp_bound_rto()
700 	return usecs_to_jiffies((tp->srtt_us >> 3) + tp->rttvar_us);  in __tcp_set_rto()
709 	tp->pred_flags = htonl((tp->tcp_header_len << 26) |  in __tcp_fast_path_on()
716 	__tcp_fast_path_on(tp, tp->snd_wnd >> tp->rx_opt.snd_wscale);  in tcp_fast_path_on()
723 	if (RB_EMPTY_ROOT(&tp->out_of_order_queue) &&  in tcp_fast_path_check()
724 	    tp->rcv_wnd &&  in tcp_fast_path_check()
725 	    atomic_read(&sk->sk_rmem_alloc) < sk->sk_rcvbuf &&  in tcp_fast_path_check()
726 	    !tp->urg_data)  in tcp_fast_path_check()
734 	u32 rto_min = inet_csk(sk)->icsk_rto_min;  in tcp_rto_min()
754 	return minmax_get(&tp->rtt_min);  in tcp_min_rtt()
763 	s32 win = tp->rcv_wup + tp->rcv_wnd - tp->rcv_nxt;  in tcp_receive_window()
780  * historically has been the same until linux-4.13.
801 /* This should only be used in contexts where tp->tcp_mstamp is up to date */
804 	return div_u64(tp->tcp_mstamp, USEC_PER_SEC / TCP_TS_HZ);  in tcp_time_stamp()
808 static inline u32 tcp_ns_to_ts(u64 ns)  in tcp_ns_to_ts()  argument
810 	return div_u64(ns, NSEC_PER_SEC / TCP_TS_HZ);  in tcp_ns_to_ts()
823 	return max_t(s64, t1 - t0, 0);  in tcp_stamp_us_delta()
828 	return tcp_ns_to_ts(skb->skb_mstamp_ns);  in tcp_skb_timestamp()
834 	return div_u64(skb->skb_mstamp_ns, NSEC_PER_USEC);  in tcp_skb_timestamp_us()
852  * TCP per-packet control information to the transmission code.
853  * We also store the host-order sequence numbers in here too.
855  * If this grows please adjust skbuff.h:skbuff->cb[xxx] size appropriately.
888 			has_rxtstamp:1,	/* SKB has a RX timestamp	*/
893 #define TCPCB_DELIVERED_CE_MASK ((1U<<20) - 1)
914 #define TCP_SKB_CB(__skb)	((struct tcp_skb_cb *)&((__skb)->cb[0]))
920  * as TCP moves IP6CB into a different location in skb->cb[]
924 	return TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_iif()
929 	bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);  in tcp_v6_iif_l3_slave()
931 	return l3_slave ? skb->skb_iif : TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_iif_l3_slave()
938 	if (skb && ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags))  in tcp_v6_sdif()
939 		return TCP_SKB_CB(skb)->header.h6.iif;  in tcp_v6_sdif()
956 	if (skb && ipv4_l3mdev_skb(TCP_SKB_CB(skb)->header.h4.flags))  in tcp_v4_sdif()
957 		return TCP_SKB_CB(skb)->header.h4.iif;  in tcp_v4_sdif()
967 	return TCP_SKB_CB(skb)->tcp_gso_segs;  in tcp_skb_pcount()
972 	TCP_SKB_CB(skb)->tcp_gso_segs = segs;  in tcp_skb_pcount_set()
977 	TCP_SKB_CB(skb)->tcp_gso_segs += segs;  in tcp_skb_pcount_add()
983 	return TCP_SKB_CB(skb)->tcp_gso_size;  in tcp_skb_mss()
988 	return likely(!TCP_SKB_CB(skb)->eor);  in tcp_skb_can_collapse_to()
1009 /* Information about inbound ACK, passed to cong_ops->in_ack_event() */
1039 /* A rate sample measures the number of (original/retransmitted) data
1041  * The tcp_rate.c code fills in the rate sample, and congestion
1043  * of ACK processing can optionally chose to consult this sample when
1045  * A sample is invalid if "delivered" or "interval_us" is negative.
1049 	u32  prior_delivered;	/* tp->delivered at "prior_mstamp" */
1050 	u32  prior_delivered_ce;/* tp->delivered_ce at "prior_mstamp" */
1053 	long interval_us;	/* time for tp->delivered to incr "delivered" */
1056 	long rtt_us;		/* RTT of last (S)ACKed packet (or -1) */
1061 	bool is_app_limited;	/* is sample from packet with bubble in pipe? */
1062 	bool is_retrans;	/* is sample from retransmission? */
1085 	void (*pkts_acked)(struct sock *sk, const struct ack_sample *sample);
1155 	return icsk->icsk_ca_ops->flags & TCP_CONG_NEEDS_ECN;  in tcp_ca_needs_ecn()
1162 	if (icsk->icsk_ca_ops->cwnd_event)  in tcp_ca_event()
1163 		icsk->icsk_ca_ops->cwnd_event(sk, event);  in tcp_ca_event()
1186  * tcp_is_sack - SACK enabled
1187  * tcp_is_reno - No SACK
1191 	return likely(tp->rx_opt.sack_ok);  in tcp_is_sack()
1201 	return tp->sacked_out + tp->lost_out;  in tcp_left_out()
1210  * tp->packets_out to determine if the send queue is empty or not.
1220 	return tp->packets_out - tcp_left_out(tp) + tp->retrans_out;  in tcp_packets_in_flight()
1227 	return tp->snd_cwnd;  in tcp_snd_cwnd()
1233 	tp->snd_cwnd = val;  in tcp_snd_cwnd_set()
1238 	return tcp_snd_cwnd(tp) < tp->snd_ssthresh;  in tcp_in_slow_start()
1243 	return tp->snd_ssthresh >= TCP_INFINITE_SSTHRESH;  in tcp_in_initial_slowstart()
1249 	       (1 << inet_csk(sk)->icsk_ca_state);  in tcp_in_cwnd_reduction()
1252 /* If cwnd > ssthresh, we may raise ssthresh to be half-way to cwnd.
1261 		return tp->snd_ssthresh;  in tcp_current_ssthresh()
1263 		return max(tp->snd_ssthresh,  in tcp_current_ssthresh()
1269 #define tcp_verify_left_out(tp)	WARN_ON(tcp_left_out(tp) > tp->packets_out)
1285 	return tp->snd_una + tp->snd_wnd;  in tcp_wnd_end()
1299  * usage, and allow application-limited process to probe bw more aggressively.
1305 	if (tp->is_cwnd_limited)  in tcp_is_cwnd_limited()
1310 		return tcp_snd_cwnd(tp) < 2 * tp->max_packets_out;  in tcp_is_cwnd_limited()
1323 	return smp_load_acquire(&sk->sk_pacing_status) == SK_PACING_NEEDED;  in tcp_needs_internal_pacing()
1331 	s64 delay = tcp_sk(sk)->tcp_wstamp_ns - tcp_sk(sk)->tcp_clock_cache;  in tcp_pacing_delay()  local
1333 	return delay > 0 ? nsecs_to_jiffies(delay) : 0;  in tcp_pacing_delay()
1353 	return max_t(unsigned long, inet_csk(sk)->icsk_rto, TCP_RTO_MIN);  in tcp_probe0_base()
1361 			   inet_csk(sk)->icsk_backoff);  in tcp_probe0_when()
1369 	if (!tcp_sk(sk)->packets_out && !inet_csk(sk)->icsk_pending)  in tcp_check_probe_timer()
1376 	tp->snd_wl1 = seq;  in tcp_init_wl()
1381 	tp->snd_wl1 = seq;  in tcp_update_wl()
1410 	rx_opt->dsack = 0;  in tcp_sack_reset()
1411 	rx_opt->num_sacks = 0;  in tcp_sack_reset()
1418 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;  in tcp_slow_start_after_idle_check()
1422 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) ||  in tcp_slow_start_after_idle_check()
1423 	    tp->packets_out || ca_ops->cong_control)  in tcp_slow_start_after_idle_check()
1425 	delta = tcp_jiffies32 - tp->lsndtime;  in tcp_slow_start_after_idle_check()
1426 	if (delta > inet_csk(sk)->icsk_rto)  in tcp_slow_start_after_idle_check()
1438 	int tcp_adv_win_scale = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_adv_win_scale);  in tcp_win_from_space()
1441 		(space>>(-tcp_adv_win_scale)) :  in tcp_win_from_space()
1442 		space - (space>>tcp_adv_win_scale);  in tcp_win_from_space()
1448 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf) -  in tcp_space()
1449 				  READ_ONCE(sk->sk_backlog.len) -  in tcp_space()
1450 				  atomic_read(&sk->sk_rmem_alloc));  in tcp_space()
1455 	return tcp_win_from_space(sk, READ_ONCE(sk->sk_rcvbuf));  in tcp_full_space()
1463 	tp->rcv_ssthresh = min(tp->rcv_ssthresh, 4U * tp->advmss);  in tcp_adjust_rcv_ssthresh()
1465 		tp->rcv_ssthresh = max_t(u32, tp->rcv_ssthresh,  in tcp_adjust_rcv_ssthresh()
1483 	rcvbuf = READ_ONCE(sk->sk_rcvbuf);  in tcp_rmem_pressure()
1484 	threshold = rcvbuf - (rcvbuf >> 3);  in tcp_rmem_pressure()
1486 	return atomic_read(&sk->sk_rmem_alloc) > threshold;  in tcp_rmem_pressure()
1492 	int avail = READ_ONCE(tp->rcv_nxt) - READ_ONCE(tp->copied_seq);  in tcp_epollin_ready()
1498 	       (tcp_receive_window(tp) <= inet_csk(sk)->icsk_ack.rcv_mss);  in tcp_epollin_ready()
1512 	return tp->keepalive_intvl ? :  in keepalive_intvl_when()
1513 		READ_ONCE(net->ipv4.sysctl_tcp_keepalive_intvl);  in keepalive_intvl_when()
1520 	return tp->keepalive_time ? :  in keepalive_time_when()
1521 		READ_ONCE(net->ipv4.sysctl_tcp_keepalive_time);  in keepalive_time_when()
1528 	return tp->keepalive_probes ? :  in keepalive_probes()
1529 		READ_ONCE(net->ipv4.sysctl_tcp_keepalive_probes);  in keepalive_probes()
1534 	const struct inet_connection_sock *icsk = &tp->inet_conn;  in keepalive_time_elapsed()
1536 	return min_t(u32, tcp_jiffies32 - icsk->icsk_ack.lrcvtime,  in keepalive_time_elapsed()
1537 			  tcp_jiffies32 - tp->rcv_tstamp);  in keepalive_time_elapsed()
1542 	int fin_timeout = tcp_sk(sk)->linger2 ? :  in tcp_fin_time()
1543 		READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fin_timeout);  in tcp_fin_time()
1544 	const int rto = inet_csk(sk)->icsk_rto;  in tcp_fin_time()
1546 	if (fin_timeout < (rto << 2) - (rto >> 1))  in tcp_fin_time()
1547 		fin_timeout = (rto << 2) - (rto >> 1);  in tcp_fin_time()
1555 	if ((s32)(rx_opt->ts_recent - rx_opt->rcv_tsval) <= paws_win)  in tcp_paws_check()
1558 				    rx_opt->ts_recent_stamp + TCP_PAWS_24DAYS)))  in tcp_paws_check()
1565 	if (!rx_opt->ts_recent)  in tcp_paws_check()
1581 	   out-of-sync and half-open connections will not be reset.  in tcp_paws_reject()
1584 	   via reboots. Linux-2.2 DOES NOT!  in tcp_paws_reject()
1589 				  rx_opt->ts_recent_stamp + TCP_PAWS_MSL))  in tcp_paws_reject()
1603 	TCP_ADD_STATS(net, TCP_MIB_MAXCONN, -1);  in tcp_mib_init()
1609 	tp->lost_skb_hint = NULL;  in tcp_clear_retrans_hints_partial()
1615 	tp->retransmit_skb_hint = NULL;  in tcp_clear_all_retrans_hints()
1625 /* - key database */
1638 /* - sock block */
1644 /* - pseudo header */
1667 /* - pool: digest algorithm, hash description and scratch buffer */
1673 /* - functions */
1705 #define tcp_twsk_md5_key(twsk)	((twsk)->tw_md5_key)
1790 	ctx = rcu_dereference(inet_csk(sk)->icsk_accept_queue.fastopenq.ctx);  in tcp_fastopen_get_ctx()
1792 		ctx = rcu_dereference(sock_net(sk)->ipv4.tcp_fastopen_ctx);  in tcp_fastopen_get_ctx()
1800 	if (orig->len == TCP_FASTOPEN_COOKIE_SIZE &&  in tcp_fastopen_cookie_match()
1801 	    orig->len == foc->len &&  in tcp_fastopen_cookie_match()
1802 	    !memcmp(orig->val, foc->val, foc->len))  in tcp_fastopen_cookie_match()
1810 	return ctx->num;  in tcp_fastopen_context_len()
1814  * chronograph-like stats that are mutually exclusive.
1818 	TCP_CHRONO_BUSY, /* Actively sending data (non-empty write queue) */
1827 /* This helper is needed, because skb->tcp_tsorted_anchor uses
1828  * the same memory storage than skb->destructor/_skb_refdst
1832 	skb->destructor = NULL;  in tcp_skb_tsorted_anchor_cleanup()
1833 	skb->_skb_refdst = 0UL;  in tcp_skb_tsorted_anchor_cleanup()
1837 	unsigned long _save = skb->_skb_refdst;	\
1838 	skb->_skb_refdst = 0UL;
1841 	skb->_skb_refdst = _save;		\
1848 	return skb_rb_first(&sk->tcp_rtx_queue);  in tcp_rtx_queue_head()
1853 	return skb_rb_last(&sk->tcp_rtx_queue);  in tcp_rtx_queue_tail()
1858 	return skb_peek_tail(&sk->sk_write_queue);  in tcp_write_queue_tail()
1862 	skb_queue_walk_from_safe(&(sk)->sk_write_queue, skb, tmp)
1866 	return skb_peek(&sk->sk_write_queue);  in tcp_send_head()
1872 	return skb_queue_is_last(&sk->sk_write_queue, skb);  in tcp_skb_is_last()
1876  * tcp_write_queue_empty - test if any payload (or FIN) is available in write queue
1880  * we must not use "return skb_queue_empty(&sk->sk_write_queue)"
1886 	return tp->write_seq == tp->snd_nxt;  in tcp_write_queue_empty()
1891 	return RB_EMPTY_ROOT(&sk->tcp_rtx_queue);  in tcp_rtx_queue_empty()
1901 	__skb_queue_tail(&sk->sk_write_queue, skb);  in tcp_add_write_queue_tail()
1904 	if (sk->sk_write_queue.next == skb)  in tcp_add_write_queue_tail()
1913 	__skb_queue_before(&sk->sk_write_queue, skb, new);  in tcp_insert_write_queue_before()
1919 	__skb_unlink(skb, &sk->sk_write_queue);  in tcp_unlink_write_queue()
1927 	rb_erase(&skb->rbnode, &sk->tcp_rtx_queue);  in tcp_rtx_queue_unlink()
1932 	list_del(&skb->tcp_tsorted_anchor);  in tcp_rtx_queue_unlink_and_free()
1942 		__tcp_push_pending_frames(sk, tcp_current_mss(sk), tp->nonagle);  in tcp_push_pending_frames()
1952 	if (!tp->sacked_out)  in tcp_highest_sack_seq()
1953 		return tp->snd_una;  in tcp_highest_sack_seq()
1955 	if (tp->highest_sack == NULL)  in tcp_highest_sack_seq()
1956 		return tp->snd_nxt;  in tcp_highest_sack_seq()
1958 	return TCP_SKB_CB(tp->highest_sack)->seq;  in tcp_highest_sack_seq()
1963 	tcp_sk(sk)->highest_sack = skb_rb_next(skb);  in tcp_advance_highest_sack()
1968 	return tcp_sk(sk)->highest_sack;  in tcp_highest_sack()
1973 	tcp_sk(sk)->highest_sack = tcp_rtx_queue_head(sk);  in tcp_highest_sack_reset()
1982 		tcp_sk(sk)->highest_sack = new;  in tcp_highest_sack_replace()
1988 	switch (sk->sk_state) {  in inet_sk_transparent()
1990 		return inet_twsk(sk)->tw_transparent;  in inet_sk_transparent()
1992 		return inet_rsk(inet_reqsk(sk))->no_srccheck;  in inet_sk_transparent()
1994 	return inet_sk(sk)->transparent;  in inet_sk_transparent()
1998  * increased latency). Used to trigger latency-reducing mechanisms.
2002 	return tp->packets_out < 4 && !tcp_in_initial_slowstart(tp);  in tcp_stream_is_thin()
2046 	return tp->notsent_lowat ?: READ_ONCE(net->ipv4.sysctl_tcp_notsent_lowat);  in tcp_notsent_lowat()
2061 /* TCP af-specific functions */
2116 	return ops->cookie_init_seq(skb, mss);  in cookie_init_sequence()
2147 	u32 rto = inet_csk(sk)->icsk_rto;  in tcp_rto_delta_us()
2150 	return rto_time_stamp_us - tcp_sk(sk)->tcp_mstamp;  in tcp_rto_delta_us()
2159 	const struct ip_options *opt = &TCP_SKB_CB(skb)->header.h4.opt;  in tcp_v4_save_options()
2162 	if (opt->optlen) {  in tcp_v4_save_options()
2163 		int opt_size = sizeof(*dopt) + opt->optlen;  in tcp_v4_save_options()
2166 		if (dopt && __ip_options_echo(net, &dopt->opt, skb, opt)) {  in tcp_v4_save_options()
2174 /* locally generated TCP pure ACKs have skb->truesize == 2
2181 	return skb->truesize == 2;  in skb_is_tcp_pure_ack()
2186 	skb->truesize = 2;  in skb_set_tcp_pure_ack()
2194 	if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV)) {  in tcp_inq()
2197 		   !tp->urg_data ||  in tcp_inq()
2198 		   before(tp->urg_seq, tp->copied_seq) ||  in tcp_inq()
2199 		   !before(tp->urg_seq, tp->rcv_nxt)) {  in tcp_inq()
2201 		answ = tp->rcv_nxt - tp->copied_seq;  in tcp_inq()
2205 			answ--;  in tcp_inq()
2207 		answ = tp->urg_seq - tp->copied_seq;  in tcp_inq()
2219 	segs_in = max_t(u16, 1, skb_shinfo(skb)->gso_segs);  in tcp_segs_in()
2224 	WRITE_ONCE(tp->segs_in, tp->segs_in + segs_in);  in tcp_segs_in()
2225 	if (skb->len > tcp_hdrlen(skb))  in tcp_segs_in()
2226 		WRITE_ONCE(tp->data_segs_in, tp->data_segs_in + segs_in);  in tcp_segs_in()
2238 	atomic_inc(&((struct sock *)sk)->sk_drops);  in tcp_listendrop()
2282 	__MODULE_INFO(alias, alias_tcp_ulp, "tcp-ulp-" name)
2309 	skops->skb = skb;  in bpf_skops_init_skb()
2310 	skops->skb_data_end = skb->data + end_offset;  in bpf_skops_init_skb()
2346 		ret = -1;  in tcp_call_bpf()
2368 	return -EPERM;  in tcp_call_bpf()
2373 	return -EPERM;  in tcp_call_bpf_2arg()
2379 	return -EPERM;  in tcp_call_bpf_3arg()
2433 		skb->skb_mstamp_ns += (u64)tp->tcp_tx_delay * NSEC_PER_USEC;  in tcp_add_tx_delay()
2442 		u32 delay = (sk->sk_state == TCP_TIME_WAIT) ?  in tcp_transmit_time()  local
2443 			tcp_twsk(sk)->tw_tx_delay : tcp_sk(sk)->tcp_tx_delay;  in tcp_transmit_time()
2445 		return tcp_clock_ns() + (u64)delay * NSEC_PER_USEC;  in tcp_transmit_time()