89 struct bbr {  struct
209 	const struct bbr *bbr = inet_csk_ca(sk);  in bbr_full_bw_reached()  local
211 	return bbr->full_bw_reached;  in bbr_full_bw_reached()
217 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_max_bw()  local
219 	return minmax_get(&bbr->bw);  in bbr_max_bw()
225 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_bw()  local
227 	return bbr->lt_use_bw ? bbr->lt_bw : bbr_max_bw(sk);  in bbr_bw()
235 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_extra_acked()  local
237 	return max(bbr->extra_acked[0], bbr->extra_acked[1]);  in bbr_extra_acked()
269 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_init_pacing_rate_from_rtt()  local
275 		bbr->has_seen_rtt = 1;  in bbr_init_pacing_rate_from_rtt()
288 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_set_pacing_rate()  local
291 	if (unlikely(!bbr->has_seen_rtt && tp->srtt_us))  in bbr_set_pacing_rate()
323 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_save_cwnd()  local
325 	if (bbr->prev_ca_state < TCP_CA_Recovery && bbr->mode != BBR_PROBE_RTT)  in bbr_save_cwnd()
326 		bbr->prior_cwnd = tcp_snd_cwnd(tp);  /* this cwnd is good enough */  in bbr_save_cwnd()
328 		bbr->prior_cwnd = max(bbr->prior_cwnd, tcp_snd_cwnd(tp));  in bbr_save_cwnd()
334 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_cwnd_event()  local
337 		bbr->idle_restart = 1;  in bbr_cwnd_event()
338 		bbr->ack_epoch_mstamp = tp->tcp_mstamp;  in bbr_cwnd_event()
339 		bbr->ack_epoch_acked = 0;  in bbr_cwnd_event()
343 		if (bbr->mode == BBR_PROBE_BW)  in bbr_cwnd_event()
345 		else if (bbr->mode == BBR_PROBE_RTT)  in bbr_cwnd_event()
361 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_bdp()  local
371 	if (unlikely(bbr->min_rtt_us == ~0U))	 /* no valid RTT samples yet? */  in bbr_bdp()
374 	w = (u64)bw * bbr->min_rtt_us;  in bbr_bdp()
396 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_quantization_budget()  local
405 	if (bbr->mode == BBR_PROBE_BW && bbr->cycle_idx == 0)  in bbr_quantization_budget()
439 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_packets_in_net_at_edt()  local
448 	if (bbr->pacing_gain > BBR_UNIT)              /* increasing inflight */  in bbr_packets_in_net_at_edt()
483 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_set_cwnd_to_recover_or_restore()  local
484 	u8 prev_state = bbr->prev_ca_state, state = inet_csk(sk)->icsk_ca_state;  in bbr_set_cwnd_to_recover_or_restore()
496 		bbr->packet_conservation = 1;  in bbr_set_cwnd_to_recover_or_restore()
497 		bbr->next_rtt_delivered = tp->delivered;  /* start round now */  in bbr_set_cwnd_to_recover_or_restore()
502 		cwnd = max(cwnd, bbr->prior_cwnd);  in bbr_set_cwnd_to_recover_or_restore()
503 		bbr->packet_conservation = 0;  in bbr_set_cwnd_to_recover_or_restore()
505 	bbr->prev_ca_state = state;  in bbr_set_cwnd_to_recover_or_restore()
507 	if (bbr->packet_conservation) {  in bbr_set_cwnd_to_recover_or_restore()
522 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_set_cwnd()  local
548 	if (bbr->mode == BBR_PROBE_RTT)  /* drain queue, refresh min_rtt */  in bbr_set_cwnd()
557 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_is_next_cycle_phase()  local
559 		tcp_stamp_us_delta(tp->delivered_mstamp, bbr->cycle_mstamp) >  in bbr_is_next_cycle_phase()
560 		bbr->min_rtt_us;  in bbr_is_next_cycle_phase()
566 	if (bbr->pacing_gain == BBR_UNIT)  in bbr_is_next_cycle_phase()
577 	if (bbr->pacing_gain > BBR_UNIT)  in bbr_is_next_cycle_phase()
580 			 inflight >= bbr_inflight(sk, bw, bbr->pacing_gain));  in bbr_is_next_cycle_phase()
593 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_advance_cycle_phase()  local
595 	bbr->cycle_idx = (bbr->cycle_idx + 1) & (CYCLE_LEN - 1);  in bbr_advance_cycle_phase()
596 	bbr->cycle_mstamp = tp->delivered_mstamp;  in bbr_advance_cycle_phase()
603 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_update_cycle_phase()  local
605 	if (bbr->mode == BBR_PROBE_BW && bbr_is_next_cycle_phase(sk, rs))  in bbr_update_cycle_phase()
611 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_reset_startup_mode()  local
613 	bbr->mode = BBR_STARTUP;  in bbr_reset_startup_mode()
618 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_reset_probe_bw_mode()  local
620 	bbr->mode = BBR_PROBE_BW;  in bbr_reset_probe_bw_mode()
621 	bbr->cycle_idx = CYCLE_LEN - 1 - prandom_u32_max(bbr_cycle_rand);  in bbr_reset_probe_bw_mode()
637 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_reset_lt_bw_sampling_interval()  local
639 	bbr->lt_last_stamp = div_u64(tp->delivered_mstamp, USEC_PER_MSEC);  in bbr_reset_lt_bw_sampling_interval()
640 	bbr->lt_last_delivered = tp->delivered;  in bbr_reset_lt_bw_sampling_interval()
641 	bbr->lt_last_lost = tp->lost;  in bbr_reset_lt_bw_sampling_interval()
642 	bbr->lt_rtt_cnt = 0;  in bbr_reset_lt_bw_sampling_interval()
648 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_reset_lt_bw_sampling()  local
650 	bbr->lt_bw = 0;  in bbr_reset_lt_bw_sampling()
651 	bbr->lt_use_bw = 0;  in bbr_reset_lt_bw_sampling()
652 	bbr->lt_is_sampling = false;  in bbr_reset_lt_bw_sampling()
659 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_lt_bw_interval_done()  local
662 	if (bbr->lt_bw) {  /* do we have bw from a previous interval? */  in bbr_lt_bw_interval_done()
664 		diff = abs(bw - bbr->lt_bw);  in bbr_lt_bw_interval_done()
665 		if ((diff * BBR_UNIT <= bbr_lt_bw_ratio * bbr->lt_bw) ||  in bbr_lt_bw_interval_done()
669 			bbr->lt_bw = (bw + bbr->lt_bw) >> 1;  /* avg 2 intvls */  in bbr_lt_bw_interval_done()
670 			bbr->lt_use_bw = 1;  in bbr_lt_bw_interval_done()
671 			bbr->pacing_gain = BBR_UNIT;  /* try to avoid drops */  in bbr_lt_bw_interval_done()
672 			bbr->lt_rtt_cnt = 0;  in bbr_lt_bw_interval_done()
676 	bbr->lt_bw = bw;  in bbr_lt_bw_interval_done()
690 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_lt_bw_sampling()  local
695 	if (bbr->lt_use_bw) {	/* already using long-term rate, lt_bw? */  in bbr_lt_bw_sampling()
696 		if (bbr->mode == BBR_PROBE_BW && bbr->round_start &&  in bbr_lt_bw_sampling()
697 		    ++bbr->lt_rtt_cnt >= bbr_lt_bw_max_rtts) {  in bbr_lt_bw_sampling()
708 	if (!bbr->lt_is_sampling) {  in bbr_lt_bw_sampling()
712 		bbr->lt_is_sampling = true;  in bbr_lt_bw_sampling()
721 	if (bbr->round_start)  in bbr_lt_bw_sampling()
722 		bbr->lt_rtt_cnt++;	/* count round trips in this interval */  in bbr_lt_bw_sampling()
723 	if (bbr->lt_rtt_cnt < bbr_lt_intvl_min_rtts)  in bbr_lt_bw_sampling()
725 	if (bbr->lt_rtt_cnt > 4 * bbr_lt_intvl_min_rtts) {  in bbr_lt_bw_sampling()
738 	lost = tp->lost - bbr->lt_last_lost;  in bbr_lt_bw_sampling()
739 	delivered = tp->delivered - bbr->lt_last_delivered;  in bbr_lt_bw_sampling()
745 	t = div_u64(tp->delivered_mstamp, USEC_PER_MSEC) - bbr->lt_last_stamp;  in bbr_lt_bw_sampling()
763 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_update_bw()  local
766 	bbr->round_start = 0;  in bbr_update_bw()
771 	if (!before(rs->prior_delivered, bbr->next_rtt_delivered)) {  in bbr_update_bw()
772 		bbr->next_rtt_delivered = tp->delivered;  in bbr_update_bw()
773 		bbr->rtt_cnt++;  in bbr_update_bw()
774 		bbr->round_start = 1;  in bbr_update_bw()
775 		bbr->packet_conservation = 0;  in bbr_update_bw()
799 		minmax_running_max(&bbr->bw, bbr_bw_rtts, bbr->rtt_cnt, bw);  in bbr_update_bw()
820 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_update_ack_aggregation()  local
827 	if (bbr->round_start) {  in bbr_update_ack_aggregation()
828 		bbr->extra_acked_win_rtts = min(0x1F,  in bbr_update_ack_aggregation()
829 						bbr->extra_acked_win_rtts + 1);  in bbr_update_ack_aggregation()
830 		if (bbr->extra_acked_win_rtts >= bbr_extra_acked_win_rtts) {  in bbr_update_ack_aggregation()
831 			bbr->extra_acked_win_rtts = 0;  in bbr_update_ack_aggregation()
832 			bbr->extra_acked_win_idx = bbr->extra_acked_win_idx ?  in bbr_update_ack_aggregation()
834 			bbr->extra_acked[bbr->extra_acked_win_idx] = 0;  in bbr_update_ack_aggregation()
840 				      bbr->ack_epoch_mstamp);  in bbr_update_ack_aggregation()
847 	if (bbr->ack_epoch_acked <= expected_acked ||  in bbr_update_ack_aggregation()
848 	    (bbr->ack_epoch_acked + rs->acked_sacked >=  in bbr_update_ack_aggregation()
850 		bbr->ack_epoch_acked = 0;  in bbr_update_ack_aggregation()
851 		bbr->ack_epoch_mstamp = tp->delivered_mstamp;  in bbr_update_ack_aggregation()
856 	bbr->ack_epoch_acked = min_t(u32, 0xFFFFF,  in bbr_update_ack_aggregation()
857 				     bbr->ack_epoch_acked + rs->acked_sacked);  in bbr_update_ack_aggregation()
858 	extra_acked = bbr->ack_epoch_acked - expected_acked;  in bbr_update_ack_aggregation()
860 	if (extra_acked > bbr->extra_acked[bbr->extra_acked_win_idx])  in bbr_update_ack_aggregation()
861 		bbr->extra_acked[bbr->extra_acked_win_idx] = extra_acked;  in bbr_update_ack_aggregation()
875 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_check_full_bw_reached()  local
878 	if (bbr_full_bw_reached(sk) || !bbr->round_start || rs->is_app_limited)  in bbr_check_full_bw_reached()
881 	bw_thresh = (u64)bbr->full_bw * bbr_full_bw_thresh >> BBR_SCALE;  in bbr_check_full_bw_reached()
883 		bbr->full_bw = bbr_max_bw(sk);  in bbr_check_full_bw_reached()
884 		bbr->full_bw_cnt = 0;  in bbr_check_full_bw_reached()
887 	++bbr->full_bw_cnt;  in bbr_check_full_bw_reached()
888 	bbr->full_bw_reached = bbr->full_bw_cnt >= bbr_full_bw_cnt;  in bbr_check_full_bw_reached()
894 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_check_drain()  local
896 	if (bbr->mode == BBR_STARTUP && bbr_full_bw_reached(sk)) {  in bbr_check_drain()
897 		bbr->mode = BBR_DRAIN;	/* drain queue we created */  in bbr_check_drain()
901 	if (bbr->mode == BBR_DRAIN &&  in bbr_check_drain()
910 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_check_probe_rtt_done()  local
912 	if (!(bbr->probe_rtt_done_stamp &&  in bbr_check_probe_rtt_done()
913 	      after(tcp_jiffies32, bbr->probe_rtt_done_stamp)))  in bbr_check_probe_rtt_done()
916 	bbr->min_rtt_stamp = tcp_jiffies32;  /* wait a while until PROBE_RTT */  in bbr_check_probe_rtt_done()
917 	tcp_snd_cwnd_set(tp, max(tcp_snd_cwnd(tp), bbr->prior_cwnd));  in bbr_check_probe_rtt_done()
943 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_update_min_rtt()  local
948 			       bbr->min_rtt_stamp + bbr_min_rtt_win_sec * HZ);  in bbr_update_min_rtt()
950 	    (rs->rtt_us < bbr->min_rtt_us ||  in bbr_update_min_rtt()
952 		bbr->min_rtt_us = rs->rtt_us;  in bbr_update_min_rtt()
953 		bbr->min_rtt_stamp = tcp_jiffies32;  in bbr_update_min_rtt()
957 	    !bbr->idle_restart && bbr->mode != BBR_PROBE_RTT) {  in bbr_update_min_rtt()
958 		bbr->mode = BBR_PROBE_RTT;  /* dip, drain queue */  in bbr_update_min_rtt()
960 		bbr->probe_rtt_done_stamp = 0;  in bbr_update_min_rtt()
963 	if (bbr->mode == BBR_PROBE_RTT) {  in bbr_update_min_rtt()
968 		if (!bbr->probe_rtt_done_stamp &&  in bbr_update_min_rtt()
970 			bbr->probe_rtt_done_stamp = tcp_jiffies32 +  in bbr_update_min_rtt()
972 			bbr->probe_rtt_round_done = 0;  in bbr_update_min_rtt()
973 			bbr->next_rtt_delivered = tp->delivered;  in bbr_update_min_rtt()
974 		} else if (bbr->probe_rtt_done_stamp) {  in bbr_update_min_rtt()
975 			if (bbr->round_start)  in bbr_update_min_rtt()
976 				bbr->probe_rtt_round_done = 1;  in bbr_update_min_rtt()
977 			if (bbr->probe_rtt_round_done)  in bbr_update_min_rtt()
983 		bbr->idle_restart = 0;  in bbr_update_min_rtt()
988 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_update_gains()  local
990 	switch (bbr->mode) {  in bbr_update_gains()
992 		bbr->pacing_gain = bbr_high_gain;  in bbr_update_gains()
993 		bbr->cwnd_gain	 = bbr_high_gain;  in bbr_update_gains()
996 		bbr->pacing_gain = bbr_drain_gain;	/* slow, to drain */  in bbr_update_gains()
997 		bbr->cwnd_gain	 = bbr_high_gain;	/* keep cwnd */  in bbr_update_gains()
1000 		bbr->pacing_gain = (bbr->lt_use_bw ?  in bbr_update_gains()
1002 				    bbr_pacing_gain[bbr->cycle_idx]);  in bbr_update_gains()
1003 		bbr->cwnd_gain	 = bbr_cwnd_gain;  in bbr_update_gains()
1006 		bbr->pacing_gain = BBR_UNIT;  in bbr_update_gains()
1007 		bbr->cwnd_gain	 = BBR_UNIT;  in bbr_update_gains()
1010 		WARN_ONCE(1, "BBR bad mode: %u\n", bbr->mode);  in bbr_update_gains()
1028 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_main()  local
1034 	bbr_set_pacing_rate(sk, bw, bbr->pacing_gain);  in bbr_main()
1035 	bbr_set_cwnd(sk, rs, rs->acked_sacked, bw, bbr->cwnd_gain);  in bbr_main()
1041 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_init()  local
1043 	bbr->prior_cwnd = 0;  in bbr_init()
1045 	bbr->rtt_cnt = 0;  in bbr_init()
1046 	bbr->next_rtt_delivered = tp->delivered;  in bbr_init()
1047 	bbr->prev_ca_state = TCP_CA_Open;  in bbr_init()
1048 	bbr->packet_conservation = 0;  in bbr_init()
1050 	bbr->probe_rtt_done_stamp = 0;  in bbr_init()
1051 	bbr->probe_rtt_round_done = 0;  in bbr_init()
1052 	bbr->min_rtt_us = tcp_min_rtt(tp);  in bbr_init()
1053 	bbr->min_rtt_stamp = tcp_jiffies32;  in bbr_init()
1055 	minmax_reset(&bbr->bw, bbr->rtt_cnt, 0);  /* init max bw to 0 */  in bbr_init()
1057 	bbr->has_seen_rtt = 0;  in bbr_init()
1060 	bbr->round_start = 0;  in bbr_init()
1061 	bbr->idle_restart = 0;  in bbr_init()
1062 	bbr->full_bw_reached = 0;  in bbr_init()
1063 	bbr->full_bw = 0;  in bbr_init()
1064 	bbr->full_bw_cnt = 0;  in bbr_init()
1065 	bbr->cycle_mstamp = 0;  in bbr_init()
1066 	bbr->cycle_idx = 0;  in bbr_init()
1070 	bbr->ack_epoch_mstamp = tp->tcp_mstamp;  in bbr_init()
1071 	bbr->ack_epoch_acked = 0;  in bbr_init()
1072 	bbr->extra_acked_win_rtts = 0;  in bbr_init()
1073 	bbr->extra_acked_win_idx = 0;  in bbr_init()
1074 	bbr->extra_acked[0] = 0;  in bbr_init()
1075 	bbr->extra_acked[1] = 0;  in bbr_init()
1091 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_undo_cwnd()  local
1093 	bbr->full_bw = 0;   /* spurious slow-down; reset full pipe detection */  in bbr_undo_cwnd()
1094 	bbr->full_bw_cnt = 0;  in bbr_undo_cwnd()
1112 		struct bbr *bbr = inet_csk_ca(sk);  in bbr_get_info()  local
1116 		memset(&info->bbr, 0, sizeof(info->bbr));  in bbr_get_info()
1117 		info->bbr.bbr_bw_lo		= (u32)bw;  in bbr_get_info()
1118 		info->bbr.bbr_bw_hi		= (u32)(bw >> 32);  in bbr_get_info()
1119 		info->bbr.bbr_min_rtt		= bbr->min_rtt_us;  in bbr_get_info()
1120 		info->bbr.bbr_pacing_gain	= bbr->pacing_gain;  in bbr_get_info()
1121 		info->bbr.bbr_cwnd_gain		= bbr->cwnd_gain;  in bbr_get_info()
1123 		return sizeof(info->bbr);  in bbr_get_info()
1130 	struct bbr *bbr = inet_csk_ca(sk);  in bbr_set_state()  local
1135 		bbr->prev_ca_state = TCP_CA_Loss;  in bbr_set_state()
1136 		bbr->full_bw = 0;  in bbr_set_state()
1137 		bbr->round_start = 1;	/* treat RTO like end of a round */  in bbr_set_state()
1181 	BUILD_BUG_ON(sizeof(struct bbr) > ICSK_CA_PRIV_SIZE);  in bbr_register()