1 // SPDX-License-Identifier: GPL-2.0-only
6  * level, it operates by taking a delivery rate sample for each ACK.
8  * A rate sample records the rate at which the network delivered packets
9  * for this flow, calculated over the time interval between the transmission
13  * the estimator generates a delivery rate sample. Typically it uses the rate
21  *    send_rate = #pkts_delivered/(last_snd_time - first_snd_time)
22  *    ack_rate  = #pkts_delivered/(last_ack_time - first_ack_time)
23  *    bw = min(send_rate, ack_rate)
28  * deliberately avoids using the inter-packet spacing approach because that
31  * TCP flows can often be application-limited in request/response workloads.
32  * The estimator marks a bandwidth sample as application-limited if there
38  * a rate sample later when the skb is (s)acked in tcp_rate_skb_delivered().
45 	  * time we received the most recent ACK, to ensure we include  in tcp_rate_skb_sent()
46 	  * the full time the network needs to deliver all in-flight  in tcp_rate_skb_sent()
53 	  * because the latter is a guess based on RTO and loss-marking  in tcp_rate_skb_sent()
55 	  * a spuriously small time interval, causing a spuriously high  in tcp_rate_skb_sent()
58 	if (!tp->packets_out) {  in tcp_rate_skb_sent()
61 		tp->first_tx_mstamp  = tstamp_us;  in tcp_rate_skb_sent()
62 		tp->delivered_mstamp = tstamp_us;  in tcp_rate_skb_sent()
65 	TCP_SKB_CB(skb)->tx.first_tx_mstamp	= tp->first_tx_mstamp;  in tcp_rate_skb_sent()
66 	TCP_SKB_CB(skb)->tx.delivered_mstamp	= tp->delivered_mstamp;  in tcp_rate_skb_sent()
67 	TCP_SKB_CB(skb)->tx.delivered		= tp->delivered;  in tcp_rate_skb_sent()
68 	TCP_SKB_CB(skb)->tx.is_app_limited	= tp->app_limited ? 1 : 0;  in tcp_rate_skb_sent()
71 /* When an skb is sacked or acked, we fill in the rate sample with the (prior)
74  * If an ACK (s)acks multiple skbs (e.g., stretched-acks), this function is
84 	if (!scb->tx.delivered_mstamp)  in tcp_rate_skb_delivered()
87 	if (!rs->prior_delivered ||  in tcp_rate_skb_delivered()
88 	    after(scb->tx.delivered, rs->prior_delivered)) {  in tcp_rate_skb_delivered()
89 		rs->prior_delivered  = scb->tx.delivered;  in tcp_rate_skb_delivered()
90 		rs->prior_mstamp     = scb->tx.delivered_mstamp;  in tcp_rate_skb_delivered()
91 		rs->is_app_limited   = scb->tx.is_app_limited;  in tcp_rate_skb_delivered()
92 		rs->is_retrans	     = scb->sacked & TCPCB_RETRANS;  in tcp_rate_skb_delivered()
94 		/* Record send time of most recently ACKed packet: */  in tcp_rate_skb_delivered()
95 		tp->first_tx_mstamp  = tcp_skb_timestamp_us(skb);  in tcp_rate_skb_delivered()
97 		rs->interval_us = tcp_stamp_us_delta(tp->first_tx_mstamp,  in tcp_rate_skb_delivered()
98 						     scb->tx.first_tx_mstamp);  in tcp_rate_skb_delivered()
105 	if (scb->sacked & TCPCB_SACKED_ACKED)  in tcp_rate_skb_delivered()
106 		scb->tx.delivered_mstamp = 0;  in tcp_rate_skb_delivered()
109 /* Update the connection delivery information and generate a rate sample. */
117 	if (tp->app_limited && after(tp->delivered, tp->app_limited))  in tcp_rate_gen()
118 		tp->app_limited = 0;  in tcp_rate_gen()
121 	 * current time. Refactor the code using a new "tcp_acktag_state"  in tcp_rate_gen()
122 	 * to carry current time, flags, stats like "tcp_sacktag_state".  in tcp_rate_gen()
125 		tp->delivered_mstamp = tp->tcp_mstamp;  in tcp_rate_gen()
127 	rs->acked_sacked = delivered;	/* freshly ACKed or SACKed */  in tcp_rate_gen()
128 	rs->losses = lost;		/* freshly marked lost */  in tcp_rate_gen()
129 	/* Return an invalid sample if no timing information is available or  in tcp_rate_gen()
134 	if (!rs->prior_mstamp || is_sack_reneg) {  in tcp_rate_gen()
135 		rs->delivered = -1;  in tcp_rate_gen()
136 		rs->interval_us = -1;  in tcp_rate_gen()
139 	rs->delivered   = tp->delivered - rs->prior_delivered;  in tcp_rate_gen()
146 	snd_us = rs->interval_us;				/* send phase */  in tcp_rate_gen()
147 	ack_us = tcp_stamp_us_delta(tp->tcp_mstamp,  in tcp_rate_gen()
148 				    rs->prior_mstamp); /* ack phase */  in tcp_rate_gen()
149 	rs->interval_us = max(snd_us, ack_us);  in tcp_rate_gen()
152 	rs->snd_interval_us = snd_us;  in tcp_rate_gen()
153 	rs->rcv_interval_us = ack_us;  in tcp_rate_gen()
155 	/* Normally we expect interval_us >= min-rtt.  in tcp_rate_gen()
156 	 * Note that rate may still be over-estimated when a spuriously  in tcp_rate_gen()
158 	 * is under-estimated (up to an RTT). However continuously  in tcp_rate_gen()
162 	if (unlikely(rs->interval_us < tcp_min_rtt(tp))) {  in tcp_rate_gen()
163 		if (!rs->is_retrans)  in tcp_rate_gen()
165 				 rs->interval_us, rs->delivered,  in tcp_rate_gen()
166 				 inet_csk(sk)->icsk_ca_state,  in tcp_rate_gen()
167 				 tp->rx_opt.sack_ok, tcp_min_rtt(tp));  in tcp_rate_gen()
168 		rs->interval_us = -1;  in tcp_rate_gen()
172 	/* Record the last non-app-limited or the highest app-limited bw */  in tcp_rate_gen()
173 	if (!rs->is_app_limited ||  in tcp_rate_gen()
174 	    ((u64)rs->delivered * tp->rate_interval_us >=  in tcp_rate_gen()
175 	     (u64)tp->rate_delivered * rs->interval_us)) {  in tcp_rate_gen()
176 		tp->rate_delivered = rs->delivered;  in tcp_rate_gen()
177 		tp->rate_interval_us = rs->interval_us;  in tcp_rate_gen()
178 		tp->rate_app_limited = rs->is_app_limited;  in tcp_rate_gen()
182 /* If a gap is detected between sends, mark the socket application-limited. */
188 	    tp->write_seq - tp->snd_nxt < tp->mss_cache &&  in tcp_rate_check_app_limited()
192 	    tcp_packets_in_flight(tp) < tp->snd_cwnd &&  in tcp_rate_check_app_limited()
194 	    tp->lost_out <= tp->retrans_out)  in tcp_rate_check_app_limited()
195 		tp->app_limited =  in tcp_rate_check_app_limited()
196 			(tp->delivered + tcp_packets_in_flight(tp)) ? : 1;  in tcp_rate_check_app_limited()