21  *    +---> STARTUP  ----+
24  *    |      DRAIN   ----+
27  *    +---> PROBE_BW ----+
30  *    |      +----+      |
32  *    +---- PROBE_RTT <--+
37  * A long-lived BBR flow spends the vast majority of its time remaining
41  * sample that matches or decreases its min_rtt estimate for 10 seconds, then
42  * it briefly enters PROBE_RTT to cut inflight to a minimum value to re-probe
43  * the path's two-way propagation delay (min_rtt). When exiting PROBE_RTT, if
48  *   "BBR: Congestion-Based Congestion Control",
50  *   Van Jacobson. ACM Queue, Vol. 14 No. 5, September-October 2016.
52  * There is a public e-mail list for discussing BBR development and testing:
53  *   https://groups.google.com/forum/#!forum/bbr-dev
55  * NOTE: BBR might be used with the fq qdisc ("man tc-fq") with pacing enabled,
83 	BBR_PROBE_RTT,	/* cut inflight to min to probe min_rtt */
88 	u32	min_rtt_us;	        /* min RTT in min_rtt_win_sec window */
90 	u32	probe_rtt_done_stamp;   /* end time for BBR_PROBE_RTT mode */
92 	u32	rtt_cnt;	    /* count of packet-timed rounds elapsed */
93 	u32     next_rtt_delivered; /* scb->tx.delivered at end of round */
94 	u64	cycle_mstamp;	     /* time of this cycle phase start */
98 		round_start:1,	     /* start of packet-timed tx->ack round? */
102 		lt_is_sampling:1,    /* taking long-term ("LT") samples now? */
103 		lt_rtt_cnt:7,	     /* round trips in long-term interval */
106 	u32	lt_last_delivered;   /* LT intvl start: tp->delivered */
107 	u32	lt_last_stamp;	     /* LT intvl start: tp->delivered_mstamp */
108 	u32	lt_last_lost;	     /* LT intvl start: tp->lost */
114 		has_seen_rtt:1, /* have we seen an RTT sample yet? */
134 /* Minimum time (in ms) spent at bbr_cwnd_min_target in BBR_PROBE_RTT mode: */
149  * and send the same number of packets per RTT that an un-paced, slow-starting
157 /* The gain for deriving steady-state cwnd tolerates delayed/stretched ACKs: */
181 /* "long-term" ("LT") bandwidth estimator parameters... */
199 /* Time period for clamping cwnd increment due to ack aggregation */
209 	return bbr->full_bw_reached;  in bbr_full_bw_reached()
212 /* Return the windowed max recent bandwidth sample, in pkts/uS << BW_SCALE. */
217 	return minmax_get(&bbr->bw);  in bbr_max_bw()
225 	return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk);  in bbr_bw()
228 /* Return maximum extra acked in past k-2k round trips,
235 	return max(bbr->extra_acked[0], bbr->extra_acked[1]);  in bbr_extra_acked()
244 	unsigned int mss = tcp_sk(sk)->mss_cache;  in bbr_rate_bytes_per_sec()
249 	rate *= USEC_PER_SEC / 100 * (100 - bbr_pacing_margin_percent);  in bbr_rate_bytes_per_sec()
259 	rate = min_t(u64, rate, sk->sk_max_pacing_rate);  in bbr_bw_to_pacing_rate()
271 	if (tp->srtt_us) {		/* any RTT sample yet? */  in bbr_init_pacing_rate_from_rtt()
272 		rtt_us = max(tp->srtt_us >> 3, 1U);  in bbr_init_pacing_rate_from_rtt()
273 		bbr->has_seen_rtt = 1;  in bbr_init_pacing_rate_from_rtt()
274 	} else {			 /* no RTT sample yet */  in bbr_init_pacing_rate_from_rtt()
277 	bw = (u64)tp->snd_cwnd * BW_UNIT;  in bbr_init_pacing_rate_from_rtt()
279 	sk->sk_pacing_rate = bbr_bw_to_pacing_rate(sk, bw, bbr_high_gain);  in bbr_init_pacing_rate_from_rtt()
289 	if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))  in bbr_set_pacing_rate()
291 	if (bbr_full_bw_reached(sk) || rate > sk->sk_pacing_rate)  in bbr_set_pacing_rate()
292 		sk->sk_pacing_rate = rate;  in bbr_set_pacing_rate()
298 	return sk->sk_pacing_rate < (bbr_min_tso_rate >> 3) ? 1 : 2;  in bbr_min_tso_segs()
310 		      sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift),  in bbr_tso_segs_goal()
311 		      GSO_MAX_SIZE - 1 - MAX_TCP_HEADER);  in bbr_tso_segs_goal()
312 	segs = max_t(u32, bytes / tp->mss_cache, bbr_min_tso_segs(sk));  in bbr_tso_segs_goal()
314 	return min(segs, 0x7FU);  in bbr_tso_segs_goal()
323 	if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT)  in bbr_save_cwnd()
324 		bbr->prior_cwnd = tp->snd_cwnd;  /* this cwnd is good enough */  in bbr_save_cwnd()
326 		bbr->prior_cwnd = max(bbr->prior_cwnd, tp->snd_cwnd);  in bbr_save_cwnd()
334 	if (event == CA_EVENT_TX_START && tp->app_limited) {  in bbr_cwnd_event()
335 		bbr->idle_restart = 1;  in bbr_cwnd_event()
336 		bbr->ack_epoch_mstamp = tp->tcp_mstamp;  in bbr_cwnd_event()
337 		bbr->ack_epoch_acked = 0;  in bbr_cwnd_event()
339 		 * need more speed (we're restarting from idle and app-limited).  in bbr_cwnd_event()
341 		if (bbr->mode == BBR_PROBE_BW)  in bbr_cwnd_event()
343 		else if (bbr->mode == BBR_PROBE_RTT)  in bbr_cwnd_event()
348 /* Calculate bdp based on min RTT and the estimated bottleneck bandwidth:
355  * noise may cause BBR to under-estimate the rate.
363 	/* If we've never had a valid RTT sample, cap cwnd at the initial  in bbr_bdp()
367 	 * case we need to slow-start up toward something safe: TCP_INIT_CWND.  in bbr_bdp()
369 	if (unlikely(bbr->min_rtt_us == ~0U))	 /* no valid RTT samples yet? */  in bbr_bdp()
372 	w = (u64)bw * bbr->min_rtt_us;  in bbr_bdp()
377 	bdp = (((w * gain) >> BBR_SCALE) + BW_UNIT - 1) / BW_UNIT;  in bbr_bdp()
382 /* To achieve full performance in high-speed paths, we budget enough cwnd to
383  * fit full-sized skbs in-flight on both end hosts to fully utilize the path:
384  *   - one skb in sending host Qdisc,
385  *   - one skb in sending host TSO/GSO engine
386  *   - one skb being received by receiver host LRO/GRO/delayed-ACK engine
389  * which allows 2 outstanding 2-packet sequences, to try to keep pipe
390  * full even with ACK-every-other-packet delayed ACKs.
396 	/* Allow enough full-sized skbs in flight to utilize end systems. */  in bbr_quantization_budget()
403 	if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0)  in bbr_quantization_budget()
409 /* Find inflight based on min RTT and the estimated bottleneck bandwidth. */
421  * "in flight". With TSQ and departure time pacing at lower layers (e.g. fq),
422  * we often have several skbs queued in the pacing layer with a pre-scheduled
423  * earliest departure time (EDT). BBR adapts its pacing rate based on the
425  * departure time decisions. We calculate a rough estimate of the number of our
426  * packets that might be in the network at the earliest departure time for the
428  *   in_network_at_edt = inflight_at_edt - (EDT - now) * bw
441 	now_ns = tp->tcp_clock_cache;  in bbr_packets_in_net_at_edt()
442 	edt_ns = max(tp->tcp_wstamp_ns, now_ns);  in bbr_packets_in_net_at_edt()
443 	interval_us = div_u64(edt_ns - now_ns, NSEC_PER_USEC);  in bbr_packets_in_net_at_edt()
446 	if (bbr->pacing_gain > BBR_UNIT)              /* increasing inflight */  in bbr_packets_in_net_at_edt()
450 	return inflight_at_edt - interval_delivered;  in bbr_packets_in_net_at_edt()
463 		aggr_cwnd = min(aggr_cwnd, max_aggr_cwnd);  in bbr_ack_aggregation_cwnd()
471  * After that, we slow-start and send at most 2*P packets per P packets acked.
475  * TODO(ycheng/ncardwell): implement a rate-based approach.
482 	u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state;  in bbr_set_cwnd_to_recover_or_restore()
483 	u32 cwnd = tp->snd_cwnd;  in bbr_set_cwnd_to_recover_or_restore()
489 	if (rs->losses > 0)  in bbr_set_cwnd_to_recover_or_restore()
490 		cwnd = max_t(s32, cwnd - rs->losses, 1);  in bbr_set_cwnd_to_recover_or_restore()
494 		bbr->packet_conservation = 1;  in bbr_set_cwnd_to_recover_or_restore()
495 		bbr->next_rtt_delivered = tp->delivered;  /* start round now */  in bbr_set_cwnd_to_recover_or_restore()
500 		cwnd = max(cwnd, bbr->prior_cwnd);  in bbr_set_cwnd_to_recover_or_restore()
501 		bbr->packet_conservation = 0;  in bbr_set_cwnd_to_recover_or_restore()
503 	bbr->prev_ca_state = state;  in bbr_set_cwnd_to_recover_or_restore()
505 	if (bbr->packet_conservation) {  in bbr_set_cwnd_to_recover_or_restore()
513 /* Slow-start up toward target cwnd (if bw estimate is growing, or packet loss
521 	u32 cwnd = tp->snd_cwnd, target_cwnd = 0;  in bbr_set_cwnd()
539 		cwnd = min(cwnd + acked, target_cwnd);  in bbr_set_cwnd()
540 	else if (cwnd < target_cwnd || tp->delivered < TCP_INIT_CWND)  in bbr_set_cwnd()
545 	tp->snd_cwnd = min(cwnd, tp->snd_cwnd_clamp);	/* apply global cap */  in bbr_set_cwnd()
546 	if (bbr->mode == BBR_PROBE_RTT)  /* drain queue, refresh min_rtt */  in bbr_set_cwnd()
547 		tp->snd_cwnd = min(tp->snd_cwnd, bbr_cwnd_min_target);  in bbr_set_cwnd()
550 /* End cycle phase if it's time and/or we hit the phase's in-flight target. */
557 		tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) >  in bbr_is_next_cycle_phase()
558 		bbr->min_rtt_us;  in bbr_is_next_cycle_phase()
564 	if (bbr->pacing_gain == BBR_UNIT)  in bbr_is_next_cycle_phase()
565 		return is_full_length;		/* just use wall clock time */  in bbr_is_next_cycle_phase()
567 	inflight = bbr_packets_in_net_at_edt(sk, rs->prior_in_flight);  in bbr_is_next_cycle_phase()
575 	if (bbr->pacing_gain > BBR_UNIT)  in bbr_is_next_cycle_phase()
577 			(rs->losses ||  /* perhaps pacing_gain*BDP won't fit */  in bbr_is_next_cycle_phase()
578 			 inflight >= bbr_inflight(sk, bw, bbr->pacing_gain));  in bbr_is_next_cycle_phase()
593 	bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1);  in bbr_advance_cycle_phase()
594 	bbr->cycle_mstamp = tp->delivered_mstamp;  in bbr_advance_cycle_phase()
603 	if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs))  in bbr_update_cycle_phase()
611 	bbr->mode = BBR_STARTUP;  in bbr_reset_startup_mode()
618 	bbr->mode = BBR_PROBE_BW;  in bbr_reset_probe_bw_mode()
619 	bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand);  in bbr_reset_probe_bw_mode()
631 /* Start a new long-term sampling interval. */
637 	bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC);  in bbr_reset_lt_bw_sampling_interval()
638 	bbr->lt_last_delivered = tp->delivered;  in bbr_reset_lt_bw_sampling_interval()
639 	bbr->lt_last_lost = tp->lost;  in bbr_reset_lt_bw_sampling_interval()
640 	bbr->lt_rtt_cnt = 0;  in bbr_reset_lt_bw_sampling_interval()
643 /* Completely reset long-term bandwidth sampling. */
648 	bbr->lt_bw = 0;  in bbr_reset_lt_bw_sampling()
649 	bbr->lt_use_bw = 0;  in bbr_reset_lt_bw_sampling()
650 	bbr->lt_is_sampling = false;  in bbr_reset_lt_bw_sampling()
654 /* Long-term bw sampling interval is done. Estimate whether we're policed. */
660 	if (bbr->lt_bw) {  /* do we have bw from a previous interval? */  in bbr_lt_bw_interval_done()
662 		diff = abs(bw - bbr->lt_bw);  in bbr_lt_bw_interval_done()
663 		if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) ||  in bbr_lt_bw_interval_done()
667 			bbr->lt_bw = (bw + bbr->lt_bw) >> 1;  /* avg 2 intvls */  in bbr_lt_bw_interval_done()
668 			bbr->lt_use_bw = 1;  in bbr_lt_bw_interval_done()
669 			bbr->pacing_gain = BBR_UNIT;  /* try to avoid drops */  in bbr_lt_bw_interval_done()
670 			bbr->lt_rtt_cnt = 0;  in bbr_lt_bw_interval_done()
674 	bbr->lt_bw = bw;  in bbr_lt_bw_interval_done()
678 /* Token-bucket traffic policers are common (see "An Internet-Wide Analysis of
679  * Traffic Policing", SIGCOMM 2016). BBR detects token-bucket policers and
683  * set lt_bw to the "long-term" average delivery rate from those 2 intervals.
693 	if (bbr->lt_use_bw) {	/* already using long-term rate, lt_bw? */  in bbr_lt_bw_sampling()
694 		if (bbr->mode == BBR_PROBE_BW && bbr->round_start &&  in bbr_lt_bw_sampling()
695 		    ++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) {  in bbr_lt_bw_sampling()
703 	 * its tokens and estimate the steady-state rate allowed by the policer.  in bbr_lt_bw_sampling()
704 	 * Starting samples earlier includes bursts that over-estimate the bw.  in bbr_lt_bw_sampling()
706 	if (!bbr->lt_is_sampling) {  in bbr_lt_bw_sampling()
707 		if (!rs->losses)  in bbr_lt_bw_sampling()
710 		bbr->lt_is_sampling = true;  in bbr_lt_bw_sampling()
714 	if (rs->is_app_limited) {  in bbr_lt_bw_sampling()
719 	if (bbr->round_start)  in bbr_lt_bw_sampling()
720 		bbr->lt_rtt_cnt++;	/* count round trips in this interval */  in bbr_lt_bw_sampling()
721 	if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts)  in bbr_lt_bw_sampling()
723 	if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) {  in bbr_lt_bw_sampling()
730 	 * tokens are exhausted under-estimates the policed rate.  in bbr_lt_bw_sampling()
732 	if (!rs->losses)  in bbr_lt_bw_sampling()
736 	lost = tp->lost - bbr->lt_last_lost;  in bbr_lt_bw_sampling()
737 	delivered = tp->delivered - bbr->lt_last_delivered;  in bbr_lt_bw_sampling()
743 	t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp;  in bbr_lt_bw_sampling()
764 	bbr->round_start = 0;  in bbr_update_bw()
765 	if (rs->delivered < 0 || rs->interval_us <= 0)  in bbr_update_bw()
769 	if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) {  in bbr_update_bw()
770 		bbr->next_rtt_delivered = tp->delivered;  in bbr_update_bw()
771 		bbr->rtt_cnt++;  in bbr_update_bw()
772 		bbr->round_start = 1;  in bbr_update_bw()
773 		bbr->packet_conservation = 0;  in bbr_update_bw()
779 	 * bandwidth sample. Delivered is in packets and interval_us in uS and  in bbr_update_bw()
782 	bw = div64_long((u64)rs->delivered * BW_UNIT, rs->interval_us);  in bbr_update_bw()
784 	/* If this sample is application-limited, it is likely to have a very  in bbr_update_bw()
786 	 * the available network rate. Such a sample could drag down estimated  in bbr_update_bw()
787 	 * bw, causing needless slow-down. Thus, to continue to send at the  in bbr_update_bw()
788 	 * last measured network rate, we filter out app-limited samples unless  in bbr_update_bw()
791 	 * So the goal during app-limited phase is to proceed with the best  in bbr_update_bw()
795 	if (!rs->is_app_limited || bw >= bbr_max_bw(sk)) {  in bbr_update_bw()
796 		/* Incorporate new sample into our max bw filter. */  in bbr_update_bw()
797 		minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw);  in bbr_update_bw()
802  * This is used to provision extra in-flight data to keep sending during
803  * inter-ACK silences.
811  * Max filter is an approximate sliding window of 5-10 (packet timed) round
821 	if (!bbr_extra_acked_gain || rs->acked_sacked <= 0 ||  in bbr_update_ack_aggregation()
822 	    rs->delivered < 0 || rs->interval_us <= 0)  in bbr_update_ack_aggregation()
825 	if (bbr->round_start) {  in bbr_update_ack_aggregation()
826 		bbr->extra_acked_win_rtts = min(0x1F,  in bbr_update_ack_aggregation()
827 						bbr->extra_acked_win_rtts + 1);  in bbr_update_ack_aggregation()
828 		if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) {  in bbr_update_ack_aggregation()
829 			bbr->extra_acked_win_rtts = 0;  in bbr_update_ack_aggregation()
830 			bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ?  in bbr_update_ack_aggregation()
832 			bbr->extra_acked[bbr->extra_acked_win_idx] = 0;  in bbr_update_ack_aggregation()
837 	epoch_us = tcp_stamp_us_delta(tp->delivered_mstamp,  in bbr_update_ack_aggregation()
838 				      bbr->ack_epoch_mstamp);  in bbr_update_ack_aggregation()
845 	if (bbr->ack_epoch_acked <= expected_acked ||  in bbr_update_ack_aggregation()
846 	    (bbr->ack_epoch_acked + rs->acked_sacked >=  in bbr_update_ack_aggregation()
848 		bbr->ack_epoch_acked = 0;  in bbr_update_ack_aggregation()
849 		bbr->ack_epoch_mstamp = tp->delivered_mstamp;  in bbr_update_ack_aggregation()
854 	bbr->ack_epoch_acked = min_t(u32, 0xFFFFF,  in bbr_update_ack_aggregation()
855 				     bbr->ack_epoch_acked + rs->acked_sacked);  in bbr_update_ack_aggregation()
856 	extra_acked = bbr->ack_epoch_acked - expected_acked;  in bbr_update_ack_aggregation()
857 	extra_acked = min(extra_acked, tp->snd_cwnd);  in bbr_update_ack_aggregation()
858 	if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])  in bbr_update_ack_aggregation()
859 		bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked;  in bbr_update_ack_aggregation()
864  * at least bbr_full_bw_thresh (25%) after bbr_full_bw_cnt (3) non-app-limited
867  * cross-traffic or radio noise can go away. CUBIC Hystart shares a similar
868  * design goal, but uses delay and inter-ACK spacing instead of bandwidth.
876 	if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited)  in bbr_check_full_bw_reached()
879 	bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE;  in bbr_check_full_bw_reached()
881 		bbr->full_bw = bbr_max_bw(sk);  in bbr_check_full_bw_reached()
882 		bbr->full_bw_cnt = 0;  in bbr_check_full_bw_reached()
885 	++bbr->full_bw_cnt;  in bbr_check_full_bw_reached()
886 	bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt;  in bbr_check_full_bw_reached()
889 /* If pipe is probably full, drain the queue and then enter steady-state. */
894 	if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {  in bbr_check_drain()
895 		bbr->mode = BBR_DRAIN;	/* drain queue we created */  in bbr_check_drain()
896 		tcp_sk(sk)->snd_ssthresh =  in bbr_check_drain()
898 	}	/* fall through to check if in-flight is already small: */  in bbr_check_drain()
899 	if (bbr->mode == BBR_DRAIN &&  in bbr_check_drain()
910 	if (!(bbr->probe_rtt_done_stamp &&  in bbr_check_probe_rtt_done()
911 	      after(tcp_jiffies32, bbr->probe_rtt_done_stamp)))  in bbr_check_probe_rtt_done()
914 	bbr->min_rtt_stamp = tcp_jiffies32;  /* wait a while until PROBE_RTT */  in bbr_check_probe_rtt_done()
915 	tp->snd_cwnd = max(tp->snd_cwnd, bbr->prior_cwnd);  in bbr_check_probe_rtt_done()
927  * After at least bbr_probe_rtt_mode_ms=200ms and at least one packet-timed
929  * re-enter the previous mode. BBR uses 200ms to approximately bound the
934  * natural silences or low-rate periods within 10 seconds where the rate is low
936  * these min RTT measurements opportunistically with our min_rtt filter. :-)
944 	/* Track min RTT seen in the min_rtt_win_sec filter window: */  in bbr_update_min_rtt()
946 			       bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ);  in bbr_update_min_rtt()
947 	if (rs->rtt_us >= 0 &&  in bbr_update_min_rtt()
948 	    (rs->rtt_us < bbr->min_rtt_us ||  in bbr_update_min_rtt()
949 	     (filter_expired && !rs->is_ack_delayed))) {  in bbr_update_min_rtt()
950 		bbr->min_rtt_us = rs->rtt_us;  in bbr_update_min_rtt()
951 		bbr->min_rtt_stamp = tcp_jiffies32;  in bbr_update_min_rtt()
955 	    !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {  in bbr_update_min_rtt()
956 		bbr->mode = BBR_PROBE_RTT;  /* dip, drain queue */  in bbr_update_min_rtt()
958 		bbr->probe_rtt_done_stamp = 0;  in bbr_update_min_rtt()
961 	if (bbr->mode == BBR_PROBE_RTT) {  in bbr_update_min_rtt()
963 		tp->app_limited =  in bbr_update_min_rtt()
964 			(tp->delivered + tcp_packets_in_flight(tp)) ? : 1;  in bbr_update_min_rtt()
965 		/* Maintain min packets in flight for max(200 ms, 1 round). */  in bbr_update_min_rtt()
966 		if (!bbr->probe_rtt_done_stamp &&  in bbr_update_min_rtt()
968 			bbr->probe_rtt_done_stamp = tcp_jiffies32 +  in bbr_update_min_rtt()
970 			bbr->probe_rtt_round_done = 0;  in bbr_update_min_rtt()
971 			bbr->next_rtt_delivered = tp->delivered;  in bbr_update_min_rtt()
972 		} else if (bbr->probe_rtt_done_stamp) {  in bbr_update_min_rtt()
973 			if (bbr->round_start)  in bbr_update_min_rtt()
974 				bbr->probe_rtt_round_done = 1;  in bbr_update_min_rtt()
975 			if (bbr->probe_rtt_round_done)  in bbr_update_min_rtt()
980 	if (rs->delivered > 0)  in bbr_update_min_rtt()
981 		bbr->idle_restart = 0;  in bbr_update_min_rtt()
988 	switch (bbr->mode) {  in bbr_update_gains()
990 		bbr->pacing_gain = bbr_high_gain;  in bbr_update_gains()
991 		bbr->cwnd_gain	 = bbr_high_gain;  in bbr_update_gains()
994 		bbr->pacing_gain = bbr_drain_gain;	/* slow, to drain */  in bbr_update_gains()
995 		bbr->cwnd_gain	 = bbr_high_gain;	/* keep cwnd */  in bbr_update_gains()
998 		bbr->pacing_gain = (bbr->lt_use_bw ?  in bbr_update_gains()
1000 				    bbr_pacing_gain[bbr->cycle_idx]);  in bbr_update_gains()
1001 		bbr->cwnd_gain	 = bbr_cwnd_gain;  in bbr_update_gains()
1004 		bbr->pacing_gain = BBR_UNIT;  in bbr_update_gains()
1005 		bbr->cwnd_gain	 = BBR_UNIT;  in bbr_update_gains()
1008 		WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode);  in bbr_update_gains()
1032 	bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);  in bbr_main()
1033 	bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);  in bbr_main()
1041 	bbr->prior_cwnd = 0;  in bbr_init()
1042 	tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;  in bbr_init()
1043 	bbr->rtt_cnt = 0;  in bbr_init()
1044 	bbr->next_rtt_delivered = 0;  in bbr_init()
1045 	bbr->prev_ca_state = TCP_CA_Open;  in bbr_init()
1046 	bbr->packet_conservation = 0;  in bbr_init()
1048 	bbr->probe_rtt_done_stamp = 0;  in bbr_init()
1049 	bbr->probe_rtt_round_done = 0;  in bbr_init()
1050 	bbr->min_rtt_us = tcp_min_rtt(tp);  in bbr_init()
1051 	bbr->min_rtt_stamp = tcp_jiffies32;  in bbr_init()
1053 	minmax_reset(&bbr->bw, bbr->rtt_cnt, 0);  /* init max bw to 0 */  in bbr_init()
1055 	bbr->has_seen_rtt = 0;  in bbr_init()
1058 	bbr->round_start = 0;  in bbr_init()
1059 	bbr->idle_restart = 0;  in bbr_init()
1060 	bbr->full_bw_reached = 0;  in bbr_init()
1061 	bbr->full_bw = 0;  in bbr_init()
1062 	bbr->full_bw_cnt = 0;  in bbr_init()
1063 	bbr->cycle_mstamp = 0;  in bbr_init()
1064 	bbr->cycle_idx = 0;  in bbr_init()
1068 	bbr->ack_epoch_mstamp = tp->tcp_mstamp;  in bbr_init()
1069 	bbr->ack_epoch_acked = 0;  in bbr_init()
1070 	bbr->extra_acked_win_rtts = 0;  in bbr_init()
1071 	bbr->extra_acked_win_idx = 0;  in bbr_init()
1072 	bbr->extra_acked[0] = 0;  in bbr_init()
1073 	bbr->extra_acked[1] = 0;  in bbr_init()
1075 	cmpxchg(&sk->sk_pacing_status, SK_PACING_NONE, SK_PACING_NEEDED);  in bbr_init()
1080 	/* Provision 3 * cwnd since BBR may slow-start even during recovery. */  in bbr_sndbuf_expand()
1091 	bbr->full_bw = 0;   /* spurious slow-down; reset full pipe detection */  in bbr_undo_cwnd()
1092 	bbr->full_bw_cnt = 0;  in bbr_undo_cwnd()
1094 	return tcp_sk(sk)->snd_cwnd;  in bbr_undo_cwnd()
1101 	return tcp_sk(sk)->snd_ssthresh;  in bbr_ssthresh()
1107 	if (ext & (1 << (INET_DIAG_BBRINFO - 1)) ||  in bbr_get_info()
1108 	    ext & (1 << (INET_DIAG_VEGASINFO - 1))) {  in bbr_get_info()
1113 		bw = bw * tp->mss_cache * USEC_PER_SEC >> BW_SCALE;  in bbr_get_info()
1114 		memset(&info->bbr, 0, sizeof(info->bbr));  in bbr_get_info()
1115 		info->bbr.bbr_bw_lo		= (u32)bw;  in bbr_get_info()
1116 		info->bbr.bbr_bw_hi		= (u32)(bw >> 32);  in bbr_get_info()
1117 		info->bbr.bbr_min_rtt		= bbr->min_rtt_us;  in bbr_get_info()
1118 		info->bbr.bbr_pacing_gain	= bbr->pacing_gain;  in bbr_get_info()
1119 		info->bbr.bbr_cwnd_gain		= bbr->cwnd_gain;  in bbr_get_info()
1121 		return sizeof(info->bbr);  in bbr_get_info()
1133 		bbr->prev_ca_state = TCP_CA_Loss;  in bbr_set_state()
1134 		bbr->full_bw = 0;  in bbr_set_state()
1135 		bbr->round_start = 1;	/* treat RTO like end of a round */  in bbr_set_state()