54 static inline struct net *tm_net(const struct tcp_metrics_block *tm)  in tm_net()  argument
57 	return READ_ONCE(tm->tcpm_net);  in tm_net()
60 static bool tcp_metric_locked(struct tcp_metrics_block *tm,  in tcp_metric_locked()  argument
64 	return READ_ONCE(tm->tcpm_lock) & (1 << idx);  in tcp_metric_locked()
67 static u32 tcp_metric_get(const struct tcp_metrics_block *tm,  in tcp_metric_get()  argument
71 	return READ_ONCE(tm->tcpm_vals[idx]);  in tcp_metric_get()
74 static void tcp_metric_set(struct tcp_metrics_block *tm,  in tcp_metric_set()  argument
79 	WRITE_ONCE(tm->tcpm_vals[idx], val);  in tcp_metric_set()
98 static void tcpm_suck_dst(struct tcp_metrics_block *tm,  in tcpm_suck_dst()  argument
105 	WRITE_ONCE(tm->tcpm_stamp, jiffies);  in tcpm_suck_dst()
119 	WRITE_ONCE(tm->tcpm_lock, val);  in tcpm_suck_dst()
122 	tcp_metric_set(tm, TCP_METRIC_RTT, msval * USEC_PER_MSEC);  in tcpm_suck_dst()
125 	tcp_metric_set(tm, TCP_METRIC_RTTVAR, msval * USEC_PER_MSEC);  in tcpm_suck_dst()
126 	tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcpm_suck_dst()
128 	tcp_metric_set(tm, TCP_METRIC_CWND,  in tcpm_suck_dst()
130 	tcp_metric_set(tm, TCP_METRIC_REORDERING,  in tcpm_suck_dst()
134 		tm->tcpm_fastopen.mss = 0;  in tcpm_suck_dst()
135 		tm->tcpm_fastopen.syn_loss = 0;  in tcpm_suck_dst()
136 		tm->tcpm_fastopen.try_exp = 0;  in tcpm_suck_dst()
137 		tm->tcpm_fastopen.cookie.exp = false;  in tcpm_suck_dst()
138 		tm->tcpm_fastopen.cookie.len = 0;  in tcpm_suck_dst()
145 static void tcpm_check_stamp(struct tcp_metrics_block *tm,  in tcpm_check_stamp()  argument
150 	if (!tm)  in tcpm_check_stamp()
152 	limit = READ_ONCE(tm->tcpm_stamp) + TCP_METRICS_TIMEOUT;  in tcpm_check_stamp()
154 		tcpm_suck_dst(tm, dst, false);  in tcpm_check_stamp()
168 	struct tcp_metrics_block *tm;  in tcpm_new()  local
178 	tm = __tcp_get_metrics(saddr, daddr, net, hash);  in tcpm_new()
179 	if (tm == TCP_METRICS_RECLAIM_PTR) {  in tcpm_new()
181 		tm = NULL;  in tcpm_new()
183 	if (tm) {  in tcpm_new()
184 		tcpm_check_stamp(tm, dst);  in tcpm_new()
192 		for (tm = deref_locked(oldest->tcpm_next); tm;  in tcpm_new()
193 		     tm = deref_locked(tm->tcpm_next)) {  in tcpm_new()
194 			if (time_before(READ_ONCE(tm->tcpm_stamp),  in tcpm_new()
196 				oldest = tm;  in tcpm_new()
198 		tm = oldest;  in tcpm_new()
200 		tm = kzalloc(sizeof(*tm), GFP_ATOMIC);  in tcpm_new()
201 		if (!tm)  in tcpm_new()
205 	WRITE_ONCE(tm->tcpm_net, net);  in tcpm_new()
207 	tm->tcpm_saddr = *saddr;  in tcpm_new()
208 	tm->tcpm_daddr = *daddr;  in tcpm_new()
210 	tcpm_suck_dst(tm, dst, reclaim);  in tcpm_new()
213 		tm->tcpm_next = tcp_metrics_hash[hash].chain;  in tcpm_new()
214 		rcu_assign_pointer(tcp_metrics_hash[hash].chain, tm);  in tcpm_new()
219 	return tm;  in tcpm_new()
222 static struct tcp_metrics_block *tcp_get_encode(struct tcp_metrics_block *tm, int depth)  in tcp_get_encode()  argument
224 	if (tm)  in tcp_get_encode()
225 		return tm;  in tcp_get_encode()
235 	struct tcp_metrics_block *tm;  in __tcp_get_metrics()  local
238 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in __tcp_get_metrics()
239 	     tm = rcu_dereference(tm->tcpm_next)) {  in __tcp_get_metrics()
240 		if (addr_same(&tm->tcpm_saddr, saddr) &&  in __tcp_get_metrics()
241 		    addr_same(&tm->tcpm_daddr, daddr) &&  in __tcp_get_metrics()
242 		    net_eq(tm_net(tm), net))  in __tcp_get_metrics()
246 	return tcp_get_encode(tm, depth);  in __tcp_get_metrics()
252 	struct tcp_metrics_block *tm;  in __tcp_get_metrics_req()  local
280 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in __tcp_get_metrics_req()
281 	     tm = rcu_dereference(tm->tcpm_next)) {  in __tcp_get_metrics_req()
282 		if (addr_same(&tm->tcpm_saddr, &saddr) &&  in __tcp_get_metrics_req()
283 		    addr_same(&tm->tcpm_daddr, &daddr) &&  in __tcp_get_metrics_req()
284 		    net_eq(tm_net(tm), net))  in __tcp_get_metrics_req()
287 	tcpm_check_stamp(tm, dst);  in __tcp_get_metrics_req()
288 	return tm;  in __tcp_get_metrics_req()
295 	struct tcp_metrics_block *tm;  in tcp_get_metrics()  local
325 	tm = __tcp_get_metrics(&saddr, &daddr, net, hash);  in tcp_get_metrics()
326 	if (tm == TCP_METRICS_RECLAIM_PTR)  in tcp_get_metrics()
327 		tm = NULL;  in tcp_get_metrics()
328 	if (!tm && create)  in tcp_get_metrics()
329 		tm = tcpm_new(dst, &saddr, &daddr, hash);  in tcp_get_metrics()
331 		tcpm_check_stamp(tm, dst);  in tcp_get_metrics()
333 	return tm;  in tcp_get_metrics()
346 	struct tcp_metrics_block *tm;  in tcp_update_metrics()  local
361 		tm = tcp_get_metrics(sk, dst, false);  in tcp_update_metrics()
362 		if (tm && !tcp_metric_locked(tm, TCP_METRIC_RTT))  in tcp_update_metrics()
363 			tcp_metric_set(tm, TCP_METRIC_RTT, 0);  in tcp_update_metrics()
366 		tm = tcp_get_metrics(sk, dst, true);  in tcp_update_metrics()
368 	if (!tm)  in tcp_update_metrics()
371 	rtt = tcp_metric_get(tm, TCP_METRIC_RTT);  in tcp_update_metrics()
378 	if (!tcp_metric_locked(tm, TCP_METRIC_RTT)) {  in tcp_update_metrics()
383 		tcp_metric_set(tm, TCP_METRIC_RTT, rtt);  in tcp_update_metrics()
386 	if (!tcp_metric_locked(tm, TCP_METRIC_RTTVAR)) {  in tcp_update_metrics()
397 		var = tcp_metric_get(tm, TCP_METRIC_RTTVAR);  in tcp_update_metrics()
403 		tcp_metric_set(tm, TCP_METRIC_RTTVAR, var);  in tcp_update_metrics()
409 		    !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {  in tcp_update_metrics()
410 			val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_update_metrics()
412 				tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
415 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
416 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
418 				tcp_metric_set(tm, TCP_METRIC_CWND,  in tcp_update_metrics()
425 		    !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH))  in tcp_update_metrics()
426 			tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
428 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
429 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
430 			tcp_metric_set(tm, TCP_METRIC_CWND, (val + tcp_snd_cwnd(tp)) >> 1);  in tcp_update_metrics()
436 		if (!tcp_metric_locked(tm, TCP_METRIC_CWND)) {  in tcp_update_metrics()
437 			val = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_update_metrics()
438 			tcp_metric_set(tm, TCP_METRIC_CWND,  in tcp_update_metrics()
442 		    !tcp_metric_locked(tm, TCP_METRIC_SSTHRESH)) {  in tcp_update_metrics()
443 			val = tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_update_metrics()
445 				tcp_metric_set(tm, TCP_METRIC_SSTHRESH,  in tcp_update_metrics()
448 		if (!tcp_metric_locked(tm, TCP_METRIC_REORDERING)) {  in tcp_update_metrics()
449 			val = tcp_metric_get(tm, TCP_METRIC_REORDERING);  in tcp_update_metrics()
453 				tcp_metric_set(tm, TCP_METRIC_REORDERING,  in tcp_update_metrics()
457 	WRITE_ONCE(tm->tcpm_stamp, jiffies);  in tcp_update_metrics()
469 	struct tcp_metrics_block *tm;  in tcp_init_metrics()  local
481 	tm = tcp_get_metrics(sk, dst, false);  in tcp_init_metrics()
482 	if (!tm) {  in tcp_init_metrics()
487 	if (tcp_metric_locked(tm, TCP_METRIC_CWND))  in tcp_init_metrics()
488 		tp->snd_cwnd_clamp = tcp_metric_get(tm, TCP_METRIC_CWND);  in tcp_init_metrics()
491 	      0 : tcp_metric_get(tm, TCP_METRIC_SSTHRESH);  in tcp_init_metrics()
497 	val = tcp_metric_get(tm, TCP_METRIC_REORDERING);  in tcp_init_metrics()
501 	crtt = tcp_metric_get(tm, TCP_METRIC_RTT);  in tcp_init_metrics()
544 	struct tcp_metrics_block *tm;  in tcp_peer_is_proven()  local
551 	tm = __tcp_get_metrics_req(req, dst);  in tcp_peer_is_proven()
552 	if (tm && tcp_metric_get(tm, TCP_METRIC_RTT))  in tcp_peer_is_proven()
564 	struct tcp_metrics_block *tm;  in tcp_fastopen_cache_get()  local
567 	tm = tcp_get_metrics(sk, __sk_dst_get(sk), false);  in tcp_fastopen_cache_get()
568 	if (tm) {  in tcp_fastopen_cache_get()
569 		struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;  in tcp_fastopen_cache_get()
589 	struct tcp_metrics_block *tm;  in tcp_fastopen_cache_set()  local
594 	tm = tcp_get_metrics(sk, dst, true);  in tcp_fastopen_cache_set()
595 	if (tm) {  in tcp_fastopen_cache_set()
596 		struct tcp_fastopen_metrics *tfom = &tm->tcpm_fastopen;  in tcp_fastopen_cache_set()
640 				 struct tcp_metrics_block *tm)  in tcp_metrics_fill_info()  argument
645 	switch (tm->tcpm_daddr.family) {  in tcp_metrics_fill_info()
648 				    inetpeer_get_addr_v4(&tm->tcpm_daddr)) < 0)  in tcp_metrics_fill_info()
651 				    inetpeer_get_addr_v4(&tm->tcpm_saddr)) < 0)  in tcp_metrics_fill_info()
656 				     inetpeer_get_addr_v6(&tm->tcpm_daddr)) < 0)  in tcp_metrics_fill_info()
659 				     inetpeer_get_addr_v6(&tm->tcpm_saddr)) < 0)  in tcp_metrics_fill_info()
667 			  jiffies - READ_ONCE(tm->tcpm_stamp),  in tcp_metrics_fill_info()
678 			u32 val = tcp_metric_get(tm, i);  in tcp_metrics_fill_info()
712 			tfom_copy[0] = tm->tcpm_fastopen;  in tcp_metrics_fill_info()
741 				 struct tcp_metrics_block *tm)  in tcp_metrics_dump_info()  argument
751 	if (tcp_metrics_fill_info(skb, tm) < 0)  in tcp_metrics_dump_info()
771 		struct tcp_metrics_block *tm;  in tcp_metrics_nl_dump()  local
775 		for (col = 0, tm = rcu_dereference(hb->chain); tm;  in tcp_metrics_nl_dump()
776 		     tm = rcu_dereference(tm->tcpm_next), col++) {  in tcp_metrics_nl_dump()
777 			if (!net_eq(tm_net(tm), net))  in tcp_metrics_nl_dump()
781 			if (tcp_metrics_dump_info(skb, cb, tm) < 0) {  in tcp_metrics_nl_dump()
839 	struct tcp_metrics_block *tm;  in tcp_metrics_nl_cmd_get()  local
869 	for (tm = rcu_dereference(tcp_metrics_hash[hash].chain); tm;  in tcp_metrics_nl_cmd_get()
870 	     tm = rcu_dereference(tm->tcpm_next)) {  in tcp_metrics_nl_cmd_get()
871 		if (addr_same(&tm->tcpm_daddr, &daddr) &&  in tcp_metrics_nl_cmd_get()
872 		    (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&  in tcp_metrics_nl_cmd_get()
873 		    net_eq(tm_net(tm), net)) {  in tcp_metrics_nl_cmd_get()
874 			ret = tcp_metrics_fill_info(msg, tm);  in tcp_metrics_nl_cmd_get()
897 	struct tcp_metrics_block *tm;  in tcp_metrics_flush_all()  local
906 		for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {  in tcp_metrics_flush_all()
907 			match = net ? net_eq(tm_net(tm), net) :  in tcp_metrics_flush_all()
908 				!refcount_read(&tm_net(tm)->ns.count);  in tcp_metrics_flush_all()
910 				rcu_assign_pointer(*pp, tm->tcpm_next);  in tcp_metrics_flush_all()
911 				kfree_rcu(tm, rcu_head);  in tcp_metrics_flush_all()
913 				pp = &tm->tcpm_next;  in tcp_metrics_flush_all()
923 	struct tcp_metrics_block *tm;  in tcp_metrics_nl_cmd_del()  local
947 	for (tm = deref_locked(*pp); tm; tm = deref_locked(*pp)) {  in tcp_metrics_nl_cmd_del()
948 		if (addr_same(&tm->tcpm_daddr, &daddr) &&  in tcp_metrics_nl_cmd_del()
949 		    (!src || addr_same(&tm->tcpm_saddr, &saddr)) &&  in tcp_metrics_nl_cmd_del()
950 		    net_eq(tm_net(tm), net)) {  in tcp_metrics_nl_cmd_del()
951 			rcu_assign_pointer(*pp, tm->tcpm_next);  in tcp_metrics_nl_cmd_del()
952 			kfree_rcu(tm, rcu_head);  in tcp_metrics_nl_cmd_del()
955 			pp = &tm->tcpm_next;  in tcp_metrics_nl_cmd_del()