52 void tcp_mstamp_refresh(struct tcp_sock *tp)  in tcp_mstamp_refresh()  argument
56 	tp->tcp_clock_cache = val;  in tcp_mstamp_refresh()
57 	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);  in tcp_mstamp_refresh()
67 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_event_new_data_sent()  local
68 	unsigned int prior_packets = tp->packets_out;  in tcp_event_new_data_sent()
70 	WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);  in tcp_event_new_data_sent()
75 	tp->packets_out += tcp_skb_pcount(skb);  in tcp_event_new_data_sent()
92 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_acceptable_seq()  local
94 	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||  in tcp_acceptable_seq()
95 	    (tp->rx_opt.wscale_ok &&  in tcp_acceptable_seq()
96 	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))  in tcp_acceptable_seq()
97 		return tp->snd_nxt;  in tcp_acceptable_seq()
99 		return tcp_wnd_end(tp);  in tcp_acceptable_seq()
118 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_advertise_mss()  local
120 	int mss = tp->advmss;  in tcp_advertise_mss()
127 			tp->advmss = mss;  in tcp_advertise_mss()
139 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_cwnd_restart()  local
140 	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));  in tcp_cwnd_restart()
141 	u32 cwnd = tp->snd_cwnd;  in tcp_cwnd_restart()
145 	tp->snd_ssthresh = tcp_current_ssthresh(sk);  in tcp_cwnd_restart()
150 	tp->snd_cwnd = max(cwnd, restart_cwnd);  in tcp_cwnd_restart()
151 	tp->snd_cwnd_stamp = tcp_jiffies32;  in tcp_cwnd_restart()
152 	tp->snd_cwnd_used = 0;  in tcp_cwnd_restart()
156 static void tcp_event_data_sent(struct tcp_sock *tp,  in tcp_event_data_sent()  argument
162 	if (tcp_packets_in_flight(tp) == 0)  in tcp_event_data_sent()
170 	if (before(tp->lsndtime, icsk->icsk_ack.lrcvtime) &&  in tcp_event_data_sent()
174 	tp->lsndtime = now;  in tcp_event_data_sent()
181 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_event_ack_sent()  local
183 	if (unlikely(tp->compressed_ack > TCP_FASTRETRANS_THRESH)) {  in tcp_event_ack_sent()
185 			      tp->compressed_ack - TCP_FASTRETRANS_THRESH);  in tcp_event_ack_sent()
186 		tp->compressed_ack = TCP_FASTRETRANS_THRESH;  in tcp_event_ack_sent()
187 		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)  in tcp_event_ack_sent()
191 	if (unlikely(rcv_nxt != tp->rcv_nxt))  in tcp_event_ack_sent()
257 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_select_window()  local
258 	u32 old_win = tp->rcv_wnd;  in tcp_select_window()
259 	u32 cur_win = tcp_receive_window(tp);  in tcp_select_window()
274 		new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);  in tcp_select_window()
276 	tp->rcv_wnd = new_win;  in tcp_select_window()
277 	tp->rcv_wup = tp->rcv_nxt;  in tcp_select_window()
282 	if (!tp->rx_opt.rcv_wscale &&  in tcp_select_window()
286 		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));  in tcp_select_window()
289 	new_win >>= tp->rx_opt.rcv_wscale;  in tcp_select_window()
293 		tp->pred_flags = 0;  in tcp_select_window()
307 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_ecn_send_synack()  local
310 	if (!(tp->ecn_flags & TCP_ECN_OK))  in tcp_ecn_send_synack()
320 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_ecn_send_syn()  local
332 	tp->ecn_flags = 0;  in tcp_ecn_send_syn()
336 		tp->ecn_flags = TCP_ECN_OK;  in tcp_ecn_send_syn()
364 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_ecn_send()  local
366 	if (tp->ecn_flags & TCP_ECN_OK) {  in tcp_ecn_send()
369 		    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {  in tcp_ecn_send()
371 			if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {  in tcp_ecn_send()
372 				tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;  in tcp_ecn_send()
380 		if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)  in tcp_ecn_send()
403 static inline bool tcp_urg_mode(const struct tcp_sock *tp)  in tcp_urg_mode()  argument
405 	return tp->snd_una != tp->snd_up;  in tcp_urg_mode()
454 static void tcp_options_write(__be32 *ptr, struct tcp_sock *tp,  in tcp_options_write()  argument
505 		struct tcp_sack_block *sp = tp->rx_opt.dsack ?  in tcp_options_write()
506 			tp->duplicate_sack : tp->selective_acks;  in tcp_options_write()
521 		tp->rx_opt.dsack = 0;  in tcp_options_write()
551 static void smc_set_option(const struct tcp_sock *tp,  in smc_set_option()  argument
557 		if (tp->syn_smc) {  in smc_set_option()
567 static void smc_set_option_cond(const struct tcp_sock *tp,  in smc_set_option_cond()  argument
574 		if (tp->syn_smc && ireq->smc_ok) {  in smc_set_option_cond()
591 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_syn_options()  local
593 	struct tcp_fastopen_request *fastopen = tp->fastopen_req;  in tcp_syn_options()
598 	    rcu_access_pointer(tp->md5sig_info)) {  in tcp_syn_options()
599 		*md5 = tp->af_specific->md5_lookup(sk, sk);  in tcp_syn_options()
621 		opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;  in tcp_syn_options()
622 		opts->tsecr = tp->rx_opt.ts_recent;  in tcp_syn_options()
626 		opts->ws = tp->rx_opt.rcv_wscale;  in tcp_syn_options()
646 			tp->syn_fastopen = 1;  in tcp_syn_options()
647 			tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;  in tcp_syn_options()
651 	smc_set_option(tp, opts, &remaining);  in tcp_syn_options()
726 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_established_options()  local
735 	    rcu_access_pointer(tp->md5sig_info)) {  in tcp_established_options()
736 		*md5 = tp->af_specific->md5_lookup(sk, sk);  in tcp_established_options()
744 	if (likely(tp->rx_opt.tstamp_ok)) {  in tcp_established_options()
746 		opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;  in tcp_established_options()
747 		opts->tsecr = tp->rx_opt.ts_recent;  in tcp_established_options()
751 	eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;  in tcp_established_options()
791 		struct tcp_sock *tp = tcp_sk(sk);  in tcp_tsq_write()  local
793 		if (tp->lost_out > tp->retrans_out &&  in tcp_tsq_write()
794 		    tp->snd_cwnd > tcp_packets_in_flight(tp)) {  in tcp_tsq_write()
795 			tcp_mstamp_refresh(tp);  in tcp_tsq_write()
799 		tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,  in tcp_tsq_write()
825 	struct tcp_sock *tp;  in tcp_tasklet_func()  local
833 		tp = list_entry(q, struct tcp_sock, tsq_node);  in tcp_tasklet_func()
834 		list_del(&tp->tsq_node);  in tcp_tasklet_func()
836 		sk = (struct sock *)tp;  in tcp_tasklet_func()
920 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_wfree()  local
954 		list_add(&tp->tsq_node, &tsq->head);  in tcp_wfree()
969 	struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);  in tcp_pace_kick()  local
970 	struct sock *sk = (struct sock *)tp;  in tcp_pace_kick()
981 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_update_skb_after_send()  local
990 		if (rate != ~0UL && rate && tp->data_segs_out >= 10) {  in tcp_update_skb_after_send()
992 			u64 credit = tp->tcp_wstamp_ns - prior_wstamp;  in tcp_update_skb_after_send()
996 			tp->tcp_wstamp_ns += len_ns;  in tcp_update_skb_after_send()
999 	list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);  in tcp_update_skb_after_send()
1018 	struct tcp_sock *tp;  in __tcp_transmit_skb()  local
1029 	tp = tcp_sk(sk);  in __tcp_transmit_skb()
1030 	prior_wstamp = tp->tcp_wstamp_ns;  in __tcp_transmit_skb()
1031 	tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);  in __tcp_transmit_skb()
1032 	skb->skb_mstamp_ns = tp->tcp_wstamp_ns;  in __tcp_transmit_skb()
1035 			- tp->snd_una;  in __tcp_transmit_skb()
1111 	if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {  in __tcp_transmit_skb()
1112 		if (before(tp->snd_up, tcb->seq + 0x10000)) {  in __tcp_transmit_skb()
1113 			th->urg_ptr = htons(tp->snd_up - tcb->seq);  in __tcp_transmit_skb()
1115 		} else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {  in __tcp_transmit_skb()
1121 	tcp_options_write((__be32 *)(th + 1), tp, &opts);  in __tcp_transmit_skb()
1130 		th->window	= htons(min(tp->rcv_wnd, 65535U));  in __tcp_transmit_skb()
1136 		tp->af_specific->calc_md5_hash(opts.hash_location,  in __tcp_transmit_skb()
1147 		tcp_event_data_sent(tp, sk);  in __tcp_transmit_skb()
1148 		tp->data_segs_out += tcp_skb_pcount(skb);  in __tcp_transmit_skb()
1149 		tp->bytes_sent += skb->len - tcp_header_size;  in __tcp_transmit_skb()
1152 	if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)  in __tcp_transmit_skb()
1156 	tp->segs_out += tcp_skb_pcount(skb);  in __tcp_transmit_skb()
1167 	tcp_add_tx_delay(skb, tp);  in __tcp_transmit_skb()
1196 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_queue_skb()  local
1199 	WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);  in tcp_queue_skb()
1226 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_adjust_pcount()  local
1228 	tp->packets_out -= decr;  in tcp_adjust_pcount()
1231 		tp->sacked_out -= decr;  in tcp_adjust_pcount()
1233 		tp->retrans_out -= decr;  in tcp_adjust_pcount()
1235 		tp->lost_out -= decr;  in tcp_adjust_pcount()
1238 	if (tcp_is_reno(tp) && decr > 0)  in tcp_adjust_pcount()
1239 		tp->sacked_out -= min_t(u32, tp->sacked_out, decr);  in tcp_adjust_pcount()
1241 	if (tp->lost_skb_hint &&  in tcp_adjust_pcount()
1242 	    before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&  in tcp_adjust_pcount()
1244 		tp->lost_cnt_hint -= decr;  in tcp_adjust_pcount()
1246 	tcp_verify_left_out(tp);  in tcp_adjust_pcount()
1299 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_fragment()  local
1373 	if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {  in tcp_fragment()
1461 	const struct tcp_sock *tp = tcp_sk(sk);  in __tcp_mtu_to_mss()  local
1479 	if (mss_now > tp->rx_opt.mss_clamp)  in __tcp_mtu_to_mss()
1480 		mss_now = tp->rx_opt.mss_clamp;  in __tcp_mtu_to_mss()
1501 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_mss_to_mtu()  local
1506 	      tp->tcp_header_len +  in tcp_mss_to_mtu()
1524 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_mtup_init()  local
1529 	icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +  in tcp_mtup_init()
1562 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_sync_mss()  local
1570 	mss_now = tcp_bound_to_half_wnd(tp, mss_now);  in tcp_sync_mss()
1576 	tp->mss_cache = mss_now;  in tcp_sync_mss()
1587 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_current_mss()  local
1594 	mss_now = tp->mss_cache;  in tcp_current_mss()
1608 	if (header_len != tp->tcp_header_len) {  in tcp_current_mss()
1609 		int delta = (int) header_len - tp->tcp_header_len;  in tcp_current_mss()
1622 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_cwnd_application_limited()  local
1627 		u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));  in tcp_cwnd_application_limited()
1628 		u32 win_used = max(tp->snd_cwnd_used, init_win);  in tcp_cwnd_application_limited()
1629 		if (win_used < tp->snd_cwnd) {  in tcp_cwnd_application_limited()
1630 			tp->snd_ssthresh = tcp_current_ssthresh(sk);  in tcp_cwnd_application_limited()
1631 			tp->snd_cwnd = (tp->snd_cwnd + win_used) >> 1;  in tcp_cwnd_application_limited()
1633 		tp->snd_cwnd_used = 0;  in tcp_cwnd_application_limited()
1635 	tp->snd_cwnd_stamp = tcp_jiffies32;  in tcp_cwnd_application_limited()
1641 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_cwnd_validate()  local
1646 	if (!before(tp->snd_una, tp->max_packets_seq) ||  in tcp_cwnd_validate()
1647 	    tp->packets_out > tp->max_packets_out) {  in tcp_cwnd_validate()
1648 		tp->max_packets_out = tp->packets_out;  in tcp_cwnd_validate()
1649 		tp->max_packets_seq = tp->snd_nxt;  in tcp_cwnd_validate()
1650 		tp->is_cwnd_limited = is_cwnd_limited;  in tcp_cwnd_validate()
1655 		tp->snd_cwnd_used = 0;  in tcp_cwnd_validate()
1656 		tp->snd_cwnd_stamp = tcp_jiffies32;  in tcp_cwnd_validate()
1659 		if (tp->packets_out > tp->snd_cwnd_used)  in tcp_cwnd_validate()
1660 			tp->snd_cwnd_used = tp->packets_out;  in tcp_cwnd_validate()
1663 		    (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&  in tcp_cwnd_validate()
1682 static bool tcp_minshall_check(const struct tcp_sock *tp)  in tcp_minshall_check()  argument
1684 	return after(tp->snd_sml, tp->snd_una) &&  in tcp_minshall_check()
1685 		!after(tp->snd_sml, tp->snd_nxt);  in tcp_minshall_check()
1696 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,  in tcp_minshall_update()  argument
1700 		tp->snd_sml = TCP_SKB_CB(skb)->end_seq;  in tcp_minshall_update()
1710 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,  in tcp_nagle_check()  argument
1715 		 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));  in tcp_nagle_check()
1763 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_mss_split_point()  local
1766 	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;  in tcp_mss_split_point()
1782 	if (tcp_nagle_check(partial != 0, tp, nonagle))  in tcp_mss_split_point()
1791 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,  in tcp_cwnd_test()  argument
1801 	in_flight = tcp_packets_in_flight(tp);  in tcp_cwnd_test()
1802 	cwnd = tp->snd_cwnd;  in tcp_cwnd_test()
1832 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,  in tcp_nagle_test()  argument
1845 	if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))  in tcp_nagle_test()
1848 	if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))  in tcp_nagle_test()
1855 static bool tcp_snd_wnd_test(const struct tcp_sock *tp,  in tcp_snd_wnd_test()  argument
1864 	return !after(end_seq, tcp_wnd_end(tp));  in tcp_snd_wnd_test()
1938 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_tso_should_defer()  local
1951 	delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;  in tcp_tso_should_defer()
1955 	in_flight = tcp_packets_in_flight(tp);  in tcp_tso_should_defer()
1958 	BUG_ON(tp->snd_cwnd <= in_flight);  in tcp_tso_should_defer()
1960 	send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;  in tcp_tso_should_defer()
1963 	cong_win = (tp->snd_cwnd - in_flight) * tp->mss_cache;  in tcp_tso_should_defer()
1968 	if (limit >= max_segs * tp->mss_cache)  in tcp_tso_should_defer()
1977 		u32 chunk = min(tp->snd_wnd, tp->snd_cwnd * tp->mss_cache);  in tcp_tso_should_defer()
1991 		if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)  in tcp_tso_should_defer()
1999 	delta = tp->tcp_clock_cache - head->tstamp;  in tcp_tso_should_defer()
2001 	if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0)  in tcp_tso_should_defer()
2036 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_mtu_check_reprobe()  local
2048 		icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +  in tcp_mtu_check_reprobe()
2088 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_mtu_probe()  local
2105 		   tp->snd_cwnd < 11 ||  in tcp_mtu_probe()
2106 		   tp->rx_opt.num_sacks || tp->rx_opt.dsack))  in tcp_mtu_probe()
2116 	size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;  in tcp_mtu_probe()
2132 	if (tp->write_seq - tp->snd_nxt < size_needed)  in tcp_mtu_probe()
2135 	if (tp->snd_wnd < size_needed)  in tcp_mtu_probe()
2137 	if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))  in tcp_mtu_probe()
2141 	if (tcp_packets_in_flight(tp) + 2 > tp->snd_cwnd) {  in tcp_mtu_probe()
2142 		if (!tcp_packets_in_flight(tp))  in tcp_mtu_probe()
2212 		tp->snd_cwnd--;  in tcp_mtu_probe()
2216 		tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;  in tcp_mtu_probe()
2217 		tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;  in tcp_mtu_probe()
2227 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_pacing_check()  local
2232 	if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)  in tcp_pacing_check()
2235 	if (!hrtimer_is_queued(&tp->pacing_timer)) {  in tcp_pacing_check()
2236 		hrtimer_start(&tp->pacing_timer,  in tcp_pacing_check()
2237 			      ns_to_ktime(tp->tcp_wstamp_ns),  in tcp_pacing_check()
2301 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)  in tcp_chrono_set()  argument
2304 	enum tcp_chrono old = tp->chrono_type;  in tcp_chrono_set()
2307 		tp->chrono_stat[old - 1] += now - tp->chrono_start;  in tcp_chrono_set()
2308 	tp->chrono_start = now;  in tcp_chrono_set()
2309 	tp->chrono_type = new;  in tcp_chrono_set()
2314 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_chrono_start()  local
2321 	if (type > tp->chrono_type)  in tcp_chrono_start()
2322 		tcp_chrono_set(tp, type);  in tcp_chrono_start()
2327 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_chrono_stop()  local
2338 		tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);  in tcp_chrono_stop()
2339 	else if (type == tp->chrono_type)  in tcp_chrono_stop()
2340 		tcp_chrono_set(tp, TCP_CHRONO_BUSY);  in tcp_chrono_stop()
2360 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_write_xmit()  local
2370 	tcp_mstamp_refresh(tp);  in tcp_write_xmit()
2385 		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {  in tcp_write_xmit()
2387 			skb->skb_mstamp_ns = tp->tcp_wstamp_ns = tp->tcp_clock_cache;  in tcp_write_xmit()
2388 			list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);  in tcp_write_xmit()
2399 		cwnd_quota = tcp_cwnd_test(tp, skb);  in tcp_write_xmit()
2408 		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {  in tcp_write_xmit()
2414 			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,  in tcp_write_xmit()
2426 		if (tso_segs > 1 && !tcp_urg_mode(tp))  in tcp_write_xmit()
2449 		tcp_minshall_update(tp, mss_now, skb);  in tcp_write_xmit()
2463 			tp->prr_out += sent_pkts;  in tcp_write_xmit()
2468 		is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tp->snd_cwnd);  in tcp_write_xmit()
2472 	return !tp->packets_out && !tcp_write_queue_empty(sk);  in tcp_write_xmit()
2478 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_schedule_loss_probe()  local
2485 	if (rcu_access_pointer(tp->fastopen_rsk))  in tcp_schedule_loss_probe()
2493 	    !tp->packets_out || !tcp_is_sack(tp) ||  in tcp_schedule_loss_probe()
2502 	if (tp->srtt_us) {  in tcp_schedule_loss_probe()
2503 		timeout = usecs_to_jiffies(tp->srtt_us >> 2);  in tcp_schedule_loss_probe()
2504 		if (tp->packets_out == 1)  in tcp_schedule_loss_probe()
2544 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_send_loss_probe()  local
2550 	if (skb && tcp_snd_wnd_test(tp, skb, mss)) {  in tcp_send_loss_probe()
2551 		pcount = tp->packets_out;  in tcp_send_loss_probe()
2553 		if (tp->packets_out > pcount)  in tcp_send_loss_probe()
2559 		WARN_ONCE(tp->packets_out,  in tcp_send_loss_probe()
2561 			  tp->packets_out, sk->sk_state, tp->snd_cwnd, mss);  in tcp_send_loss_probe()
2567 	if (tp->tlp_high_seq)  in tcp_send_loss_probe()
2592 	tp->tlp_high_seq = tp->snd_nxt;  in tcp_send_loss_probe()
2688 	struct tcp_sock *tp = tcp_sk(sk);  in __tcp_select_window()  local
2698 	int full_space = min_t(int, tp->window_clamp, allowed_space);  in __tcp_select_window()
2710 			tp->rcv_ssthresh = min(tp->rcv_ssthresh,  in __tcp_select_window()
2711 					       4U * tp->advmss);  in __tcp_select_window()
2716 		free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);  in __tcp_select_window()
2729 	if (free_space > tp->rcv_ssthresh)  in __tcp_select_window()
2730 		free_space = tp->rcv_ssthresh;  in __tcp_select_window()
2735 	if (tp->rx_opt.rcv_wscale) {  in __tcp_select_window()
2742 		window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));  in __tcp_select_window()
2744 		window = tp->rcv_wnd;  in __tcp_select_window()
2781 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_collapse_retrans()  local
2811 	tcp_clear_retrans_hints_partial(tp);  in tcp_collapse_retrans()
2812 	if (next_skb == tp->retransmit_skb_hint)  in tcp_collapse_retrans()
2813 		tp->retransmit_skb_hint = skb;  in tcp_collapse_retrans()
2843 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_retrans_try_collapse()  local
2869 		if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))  in tcp_retrans_try_collapse()
2884 	struct tcp_sock *tp = tcp_sk(sk);  in __tcp_retransmit_skb()  local
2904 	if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {  in __tcp_retransmit_skb()
2905 		if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {  in __tcp_retransmit_skb()
2909 		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))  in __tcp_retransmit_skb()
2923 	if (!before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp)) &&  in __tcp_retransmit_skb()
2924 	    TCP_SKB_CB(skb)->seq != tp->snd_una)  in __tcp_retransmit_skb()
2954 	tp->total_retrans += segs;  in __tcp_retransmit_skb()
2955 	tp->bytes_retrans += skb->len;  in __tcp_retransmit_skb()
2972 			tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);  in __tcp_retransmit_skb()
2984 	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))  in __tcp_retransmit_skb()
2998 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_retransmit_skb()  local
3008 		tp->retrans_out += tcp_skb_pcount(skb);  in tcp_retransmit_skb()
3012 	if (!tp->retrans_stamp)  in tcp_retransmit_skb()
3013 		tp->retrans_stamp = tcp_skb_timestamp(skb);  in tcp_retransmit_skb()
3015 	if (tp->undo_retrans < 0)  in tcp_retransmit_skb()
3016 		tp->undo_retrans = 0;  in tcp_retransmit_skb()
3017 	tp->undo_retrans += tcp_skb_pcount(skb);  in tcp_retransmit_skb()
3030 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_xmit_retransmit_queue()  local
3034 	if (!tp->packets_out)  in tcp_xmit_retransmit_queue()
3038 	skb = tp->retransmit_skb_hint ?: rtx_head;  in tcp_xmit_retransmit_queue()
3049 			tp->retransmit_skb_hint = skb;  in tcp_xmit_retransmit_queue()
3051 		segs = tp->snd_cwnd - tcp_packets_in_flight(tp);  in tcp_xmit_retransmit_queue()
3060 		if (tp->retrans_out >= tp->lost_out) {  in tcp_xmit_retransmit_queue()
3086 			tp->prr_out += tcp_skb_pcount(skb);  in tcp_xmit_retransmit_queue()
3124 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_send_fin()  local
3137 		tp->write_seq++;  in tcp_send_fin()
3145 			WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);  in tcp_send_fin()
3157 		tcp_init_nondata_skb(skb, tp->write_seq,  in tcp_send_fin()
3251 	const struct tcp_sock *tp = tcp_sk(sk);  in tcp_make_synack()  local
3287 	mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));  in tcp_make_synack()
3341 	tcp_add_tx_delay(skb, tp);  in tcp_make_synack()
3370 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_connect_init()  local
3377 	tp->tcp_header_len = sizeof(struct tcphdr);  in tcp_connect_init()
3379 		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;  in tcp_connect_init()
3382 	if (tp->af_specific->md5_lookup(sk, sk))  in tcp_connect_init()
3383 		tp->tcp_header_len += TCPOLEN_MD5SIG_ALIGNED;  in tcp_connect_init()
3387 	if (tp->rx_opt.user_mss)  in tcp_connect_init()
3388 		tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;  in tcp_connect_init()
3389 	tp->max_window = 0;  in tcp_connect_init()
3395 	if (!tp->window_clamp)  in tcp_connect_init()
3396 		tp->window_clamp = dst_metric(dst, RTAX_WINDOW);  in tcp_connect_init()
3397 	tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));  in tcp_connect_init()
3403 	    (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))  in tcp_connect_init()
3404 		tp->window_clamp = tcp_full_space(sk);  in tcp_connect_init()
3411 				  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),  in tcp_connect_init()
3412 				  &tp->rcv_wnd,  in tcp_connect_init()
3413 				  &tp->window_clamp,  in tcp_connect_init()
3418 	tp->rx_opt.rcv_wscale = rcv_wscale;  in tcp_connect_init()
3419 	tp->rcv_ssthresh = tp->rcv_wnd;  in tcp_connect_init()
3423 	tp->snd_wnd = 0;  in tcp_connect_init()
3424 	tcp_init_wl(tp, 0);  in tcp_connect_init()
3426 	tp->snd_una = tp->write_seq;  in tcp_connect_init()
3427 	tp->snd_sml = tp->write_seq;  in tcp_connect_init()
3428 	tp->snd_up = tp->write_seq;  in tcp_connect_init()
3429 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);  in tcp_connect_init()
3431 	if (likely(!tp->repair))  in tcp_connect_init()
3432 		tp->rcv_nxt = 0;  in tcp_connect_init()
3434 		tp->rcv_tstamp = tcp_jiffies32;  in tcp_connect_init()
3435 	tp->rcv_wup = tp->rcv_nxt;  in tcp_connect_init()
3436 	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);  in tcp_connect_init()
3440 	tcp_clear_retrans(tp);  in tcp_connect_init()
3445 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_connect_queue_skb()  local
3452 	WRITE_ONCE(tp->write_seq, tcb->end_seq);  in tcp_connect_queue_skb()
3453 	tp->packets_out += tcp_skb_pcount(skb);  in tcp_connect_queue_skb()
3465 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_send_syn_data()  local
3466 	struct tcp_fastopen_request *fo = tp->fastopen_req;  in tcp_send_syn_data()
3470 	tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */  in tcp_send_syn_data()
3471 	if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))  in tcp_send_syn_data()
3478 	tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);  in tcp_send_syn_data()
3528 		tp->syn_data = (fo->copied > 0);  in tcp_send_syn_data()
3536 	tp->packets_out -= tcp_skb_pcount(syn_data);  in tcp_send_syn_data()
3544 		tp->syn_fastopen = 0;  in tcp_send_syn_data()
3553 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_connect()  local
3564 	if (unlikely(tp->repair)) {  in tcp_connect()
3573 	tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);  in tcp_connect()
3574 	tcp_mstamp_refresh(tp);  in tcp_connect()
3575 	tp->retrans_stamp = tcp_time_stamp(tp);  in tcp_connect()
3581 	err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :  in tcp_connect()
3589 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);  in tcp_connect()
3590 	tp->pushed_seq = tp->write_seq;  in tcp_connect()
3593 		WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);  in tcp_connect()
3594 		tp->pushed_seq	= TCP_SKB_CB(buff)->seq;  in tcp_connect()
3616 		const struct tcp_sock *tp = tcp_sk(sk);  in tcp_send_delayed_ack()  local
3629 		if (tp->srtt_us) {  in tcp_send_delayed_ack()
3630 			int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),  in tcp_send_delayed_ack()
3718 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_xmit_probe_skb()  local
3733 	tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);  in tcp_xmit_probe_skb()
3751 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_write_wakeup()  local
3758 	if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {  in tcp_write_wakeup()
3761 		unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;  in tcp_write_wakeup()
3763 		if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))  in tcp_write_wakeup()
3764 			tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;  in tcp_write_wakeup()
3786 		if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))  in tcp_write_wakeup()
3798 	struct tcp_sock *tp = tcp_sk(sk);  in tcp_send_probe0()  local
3805 	if (tp->packets_out || tcp_write_queue_empty(sk)) {  in tcp_send_probe0()