1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Implementation of the Transmission Control Protocol(TCP).
8  *
9  * Authors:	Ross Biro
10  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11  *		Mark Evans, <evansmp@uhura.aston.ac.uk>
12  *		Corey Minyard <wf-rch!minyard@relay.EU.net>
13  *		Florian La Roche, <flla@stud.uni-sb.de>
14  *		Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15  *		Linus Torvalds, <torvalds@cs.helsinki.fi>
16  *		Alan Cox, <gw4pts@gw4pts.ampr.org>
17  *		Matthew Dillon, <dillon@apollo.west.oic.com>
18  *		Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19  *		Jorge Cwik, <jorge@laser.satlink.net>
20  */
21 
22 /*
23  * Changes:	Pedro Roque	:	Retransmit queue handled by TCP.
24  *				:	Fragmentation on mtu decrease
25  *				:	Segment collapse on retransmit
26  *				:	AF independence
27  *
28  *		Linus Torvalds	:	send_delayed_ack
29  *		David S. Miller	:	Charge memory using the right skb
30  *					during syn/ack processing.
31  *		David S. Miller :	Output engine completely rewritten.
32  *		Andrea Arcangeli:	SYNACK carry ts_recent in tsecr.
33  *		Cacophonix Gaul :	draft-minshall-nagle-01
34  *		J Hadi Salim	:	ECN support
35  *
36  */
37 
38 #define pr_fmt(fmt) "TCP: " fmt
39 
40 #include <net/tcp.h>
41 #include <net/mptcp.h>
42 
43 #include <linux/compiler.h>
44 #include <linux/gfp.h>
45 #include <linux/module.h>
46 #include <linux/static_key.h>
47 
48 #include <trace/events/tcp.h>
49 
50 /* Refresh clocks of a TCP socket,
51  * ensuring monotically increasing values.
52  */
tcp_mstamp_refresh(struct tcp_sock * tp)53 void tcp_mstamp_refresh(struct tcp_sock *tp)
54 {
55 	u64 val = tcp_clock_ns();
56 
57 	tp->tcp_clock_cache = val;
58 	tp->tcp_mstamp = div_u64(val, NSEC_PER_USEC);
59 }
60 
61 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
62 			   int push_one, gfp_t gfp);
63 
64 /* Account for new data that has been sent to the network. */
tcp_event_new_data_sent(struct sock * sk,struct sk_buff * skb)65 static void tcp_event_new_data_sent(struct sock *sk, struct sk_buff *skb)
66 {
67 	struct inet_connection_sock *icsk = inet_csk(sk);
68 	struct tcp_sock *tp = tcp_sk(sk);
69 	unsigned int prior_packets = tp->packets_out;
70 
71 	WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(skb)->end_seq);
72 
73 	__skb_unlink(skb, &sk->sk_write_queue);
74 	tcp_rbtree_insert(&sk->tcp_rtx_queue, skb);
75 
76 	if (tp->highest_sack == NULL)
77 		tp->highest_sack = skb;
78 
79 	tp->packets_out += tcp_skb_pcount(skb);
80 	if (!prior_packets || icsk->icsk_pending == ICSK_TIME_LOSS_PROBE)
81 		tcp_rearm_rto(sk);
82 
83 	NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT,
84 		      tcp_skb_pcount(skb));
85 	tcp_check_space(sk);
86 }
87 
88 /* SND.NXT, if window was not shrunk or the amount of shrunk was less than one
89  * window scaling factor due to loss of precision.
90  * If window has been shrunk, what should we make? It is not clear at all.
91  * Using SND.UNA we will fail to open window, SND.NXT is out of window. :-(
92  * Anything in between SND.UNA...SND.UNA+SND.WND also can be already
93  * invalid. OK, let's make this for now:
94  */
tcp_acceptable_seq(const struct sock * sk)95 static inline __u32 tcp_acceptable_seq(const struct sock *sk)
96 {
97 	const struct tcp_sock *tp = tcp_sk(sk);
98 
99 	if (!before(tcp_wnd_end(tp), tp->snd_nxt) ||
100 	    (tp->rx_opt.wscale_ok &&
101 	     ((tp->snd_nxt - tcp_wnd_end(tp)) < (1 << tp->rx_opt.rcv_wscale))))
102 		return tp->snd_nxt;
103 	else
104 		return tcp_wnd_end(tp);
105 }
106 
107 /* Calculate mss to advertise in SYN segment.
108  * RFC1122, RFC1063, draft-ietf-tcpimpl-pmtud-01 state that:
109  *
110  * 1. It is independent of path mtu.
111  * 2. Ideally, it is maximal possible segment size i.e. 65535-40.
112  * 3. For IPv4 it is reasonable to calculate it from maximal MTU of
113  *    attached devices, because some buggy hosts are confused by
114  *    large MSS.
115  * 4. We do not make 3, we advertise MSS, calculated from first
116  *    hop device mtu, but allow to raise it to ip_rt_min_advmss.
117  *    This may be overridden via information stored in routing table.
118  * 5. Value 65535 for MSS is valid in IPv6 and means "as large as possible,
119  *    probably even Jumbo".
120  */
tcp_advertise_mss(struct sock * sk)121 static __u16 tcp_advertise_mss(struct sock *sk)
122 {
123 	struct tcp_sock *tp = tcp_sk(sk);
124 	const struct dst_entry *dst = __sk_dst_get(sk);
125 	int mss = tp->advmss;
126 
127 	if (dst) {
128 		unsigned int metric = dst_metric_advmss(dst);
129 
130 		if (metric < mss) {
131 			mss = metric;
132 			tp->advmss = mss;
133 		}
134 	}
135 
136 	return (__u16)mss;
137 }
138 
139 /* RFC2861. Reset CWND after idle period longer RTO to "restart window".
140  * This is the first part of cwnd validation mechanism.
141  */
tcp_cwnd_restart(struct sock * sk,s32 delta)142 void tcp_cwnd_restart(struct sock *sk, s32 delta)
143 {
144 	struct tcp_sock *tp = tcp_sk(sk);
145 	u32 restart_cwnd = tcp_init_cwnd(tp, __sk_dst_get(sk));
146 	u32 cwnd = tcp_snd_cwnd(tp);
147 
148 	tcp_ca_event(sk, CA_EVENT_CWND_RESTART);
149 
150 	tp->snd_ssthresh = tcp_current_ssthresh(sk);
151 	restart_cwnd = min(restart_cwnd, cwnd);
152 
153 	while ((delta -= inet_csk(sk)->icsk_rto) > 0 && cwnd > restart_cwnd)
154 		cwnd >>= 1;
155 	tcp_snd_cwnd_set(tp, max(cwnd, restart_cwnd));
156 	tp->snd_cwnd_stamp = tcp_jiffies32;
157 	tp->snd_cwnd_used = 0;
158 }
159 
160 /* Congestion state accounting after a packet has been sent. */
tcp_event_data_sent(struct tcp_sock * tp,struct sock * sk)161 static void tcp_event_data_sent(struct tcp_sock *tp,
162 				struct sock *sk)
163 {
164 	struct inet_connection_sock *icsk = inet_csk(sk);
165 	const u32 now = tcp_jiffies32;
166 
167 	if (tcp_packets_in_flight(tp) == 0)
168 		tcp_ca_event(sk, CA_EVENT_TX_START);
169 
170 	tp->lsndtime = now;
171 
172 	/* If it is a reply for ato after last received
173 	 * packet, enter pingpong mode.
174 	 */
175 	if ((u32)(now - icsk->icsk_ack.lrcvtime) < icsk->icsk_ack.ato)
176 		inet_csk_enter_pingpong_mode(sk);
177 }
178 
179 /* Account for an ACK we sent. */
tcp_event_ack_sent(struct sock * sk,u32 rcv_nxt)180 static inline void tcp_event_ack_sent(struct sock *sk, u32 rcv_nxt)
181 {
182 	struct tcp_sock *tp = tcp_sk(sk);
183 
184 	if (unlikely(tp->compressed_ack)) {
185 		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPACKCOMPRESSED,
186 			      tp->compressed_ack);
187 		tp->compressed_ack = 0;
188 		if (hrtimer_try_to_cancel(&tp->compressed_ack_timer) == 1)
189 			__sock_put(sk);
190 	}
191 
192 	if (unlikely(rcv_nxt != tp->rcv_nxt))
193 		return;  /* Special ACK sent by DCTCP to reflect ECN */
194 	tcp_dec_quickack_mode(sk);
195 	inet_csk_clear_xmit_timer(sk, ICSK_TIME_DACK);
196 }
197 
198 /* Determine a window scaling and initial window to offer.
199  * Based on the assumption that the given amount of space
200  * will be offered. Store the results in the tp structure.
201  * NOTE: for smooth operation initial space offering should
202  * be a multiple of mss if possible. We assume here that mss >= 1.
203  * This MUST be enforced by all callers.
204  */
tcp_select_initial_window(const struct sock * sk,int __space,__u32 mss,__u32 * rcv_wnd,__u32 * window_clamp,int wscale_ok,__u8 * rcv_wscale,__u32 init_rcv_wnd)205 void tcp_select_initial_window(const struct sock *sk, int __space, __u32 mss,
206 			       __u32 *rcv_wnd, __u32 *window_clamp,
207 			       int wscale_ok, __u8 *rcv_wscale,
208 			       __u32 init_rcv_wnd)
209 {
210 	unsigned int space = (__space < 0 ? 0 : __space);
211 
212 	/* If no clamp set the clamp to the max possible scaled window */
213 	if (*window_clamp == 0)
214 		(*window_clamp) = (U16_MAX << TCP_MAX_WSCALE);
215 	space = min(*window_clamp, space);
216 
217 	/* Quantize space offering to a multiple of mss if possible. */
218 	if (space > mss)
219 		space = rounddown(space, mss);
220 
221 	/* NOTE: offering an initial window larger than 32767
222 	 * will break some buggy TCP stacks. If the admin tells us
223 	 * it is likely we could be speaking with such a buggy stack
224 	 * we will truncate our initial window offering to 32K-1
225 	 * unless the remote has sent us a window scaling option,
226 	 * which we interpret as a sign the remote TCP is not
227 	 * misinterpreting the window field as a signed quantity.
228 	 */
229 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_workaround_signed_windows))
230 		(*rcv_wnd) = min(space, MAX_TCP_WINDOW);
231 	else
232 		(*rcv_wnd) = min_t(u32, space, U16_MAX);
233 
234 	if (init_rcv_wnd)
235 		*rcv_wnd = min(*rcv_wnd, init_rcv_wnd * mss);
236 
237 	*rcv_wscale = 0;
238 	if (wscale_ok) {
239 		/* Set window scaling on max possible window */
240 		space = max_t(u32, space, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]));
241 		space = max_t(u32, space, READ_ONCE(sysctl_rmem_max));
242 		space = min_t(u32, space, *window_clamp);
243 		*rcv_wscale = clamp_t(int, ilog2(space) - 15,
244 				      0, TCP_MAX_WSCALE);
245 	}
246 	/* Set the clamp no higher than max representable value */
247 	(*window_clamp) = min_t(__u32, U16_MAX << (*rcv_wscale), *window_clamp);
248 }
249 EXPORT_SYMBOL(tcp_select_initial_window);
250 
251 /* Chose a new window to advertise, update state in tcp_sock for the
252  * socket, and return result with RFC1323 scaling applied.  The return
253  * value can be stuffed directly into th->window for an outgoing
254  * frame.
255  */
tcp_select_window(struct sock * sk)256 static u16 tcp_select_window(struct sock *sk)
257 {
258 	struct tcp_sock *tp = tcp_sk(sk);
259 	struct net *net = sock_net(sk);
260 	u32 old_win = tp->rcv_wnd;
261 	u32 cur_win, new_win;
262 
263 	/* Make the window 0 if we failed to queue the data because we
264 	 * are out of memory. The window is temporary, so we don't store
265 	 * it on the socket.
266 	 */
267 	if (unlikely(inet_csk(sk)->icsk_ack.pending & ICSK_ACK_NOMEM))
268 		return 0;
269 
270 	cur_win = tcp_receive_window(tp);
271 	new_win = __tcp_select_window(sk);
272 	if (new_win < cur_win) {
273 		/* Danger Will Robinson!
274 		 * Don't update rcv_wup/rcv_wnd here or else
275 		 * we will not be able to advertise a zero
276 		 * window in time.  --DaveM
277 		 *
278 		 * Relax Will Robinson.
279 		 */
280 		if (!READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) || !tp->rx_opt.rcv_wscale) {
281 			/* Never shrink the offered window */
282 			if (new_win == 0)
283 				NET_INC_STATS(net, LINUX_MIB_TCPWANTZEROWINDOWADV);
284 			new_win = ALIGN(cur_win, 1 << tp->rx_opt.rcv_wscale);
285 		}
286 	}
287 
288 	tp->rcv_wnd = new_win;
289 	tp->rcv_wup = tp->rcv_nxt;
290 
291 	/* Make sure we do not exceed the maximum possible
292 	 * scaled window.
293 	 */
294 	if (!tp->rx_opt.rcv_wscale &&
295 	    READ_ONCE(net->ipv4.sysctl_tcp_workaround_signed_windows))
296 		new_win = min(new_win, MAX_TCP_WINDOW);
297 	else
298 		new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
299 
300 	/* RFC1323 scaling applied */
301 	new_win >>= tp->rx_opt.rcv_wscale;
302 
303 	/* If we advertise zero window, disable fast path. */
304 	if (new_win == 0) {
305 		tp->pred_flags = 0;
306 		if (old_win)
307 			NET_INC_STATS(net, LINUX_MIB_TCPTOZEROWINDOWADV);
308 	} else if (old_win == 0) {
309 		NET_INC_STATS(net, LINUX_MIB_TCPFROMZEROWINDOWADV);
310 	}
311 
312 	return new_win;
313 }
314 
315 /* Packet ECN state for a SYN-ACK */
tcp_ecn_send_synack(struct sock * sk,struct sk_buff * skb)316 static void tcp_ecn_send_synack(struct sock *sk, struct sk_buff *skb)
317 {
318 	const struct tcp_sock *tp = tcp_sk(sk);
319 
320 	TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_CWR;
321 	if (!(tp->ecn_flags & TCP_ECN_OK))
322 		TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_ECE;
323 	else if (tcp_ca_needs_ecn(sk) ||
324 		 tcp_bpf_ca_needs_ecn(sk))
325 		INET_ECN_xmit(sk);
326 }
327 
328 /* Packet ECN state for a SYN.  */
tcp_ecn_send_syn(struct sock * sk,struct sk_buff * skb)329 static void tcp_ecn_send_syn(struct sock *sk, struct sk_buff *skb)
330 {
331 	struct tcp_sock *tp = tcp_sk(sk);
332 	bool bpf_needs_ecn = tcp_bpf_ca_needs_ecn(sk);
333 	bool use_ecn = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn) == 1 ||
334 		tcp_ca_needs_ecn(sk) || bpf_needs_ecn;
335 
336 	if (!use_ecn) {
337 		const struct dst_entry *dst = __sk_dst_get(sk);
338 
339 		if (dst && dst_feature(dst, RTAX_FEATURE_ECN))
340 			use_ecn = true;
341 	}
342 
343 	tp->ecn_flags = 0;
344 
345 	if (use_ecn) {
346 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ECE | TCPHDR_CWR;
347 		tp->ecn_flags = TCP_ECN_OK;
348 		if (tcp_ca_needs_ecn(sk) || bpf_needs_ecn)
349 			INET_ECN_xmit(sk);
350 	}
351 }
352 
tcp_ecn_clear_syn(struct sock * sk,struct sk_buff * skb)353 static void tcp_ecn_clear_syn(struct sock *sk, struct sk_buff *skb)
354 {
355 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_ecn_fallback))
356 		/* tp->ecn_flags are cleared at a later point in time when
357 		 * SYN ACK is ultimatively being received.
358 		 */
359 		TCP_SKB_CB(skb)->tcp_flags &= ~(TCPHDR_ECE | TCPHDR_CWR);
360 }
361 
362 static void
tcp_ecn_make_synack(const struct request_sock * req,struct tcphdr * th)363 tcp_ecn_make_synack(const struct request_sock *req, struct tcphdr *th)
364 {
365 	if (inet_rsk(req)->ecn_ok)
366 		th->ece = 1;
367 }
368 
369 /* Set up ECN state for a packet on a ESTABLISHED socket that is about to
370  * be sent.
371  */
tcp_ecn_send(struct sock * sk,struct sk_buff * skb,struct tcphdr * th,int tcp_header_len)372 static void tcp_ecn_send(struct sock *sk, struct sk_buff *skb,
373 			 struct tcphdr *th, int tcp_header_len)
374 {
375 	struct tcp_sock *tp = tcp_sk(sk);
376 
377 	if (tp->ecn_flags & TCP_ECN_OK) {
378 		/* Not-retransmitted data segment: set ECT and inject CWR. */
379 		if (skb->len != tcp_header_len &&
380 		    !before(TCP_SKB_CB(skb)->seq, tp->snd_nxt)) {
381 			INET_ECN_xmit(sk);
382 			if (tp->ecn_flags & TCP_ECN_QUEUE_CWR) {
383 				tp->ecn_flags &= ~TCP_ECN_QUEUE_CWR;
384 				th->cwr = 1;
385 				skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN;
386 			}
387 		} else if (!tcp_ca_needs_ecn(sk)) {
388 			/* ACK or retransmitted segment: clear ECT|CE */
389 			INET_ECN_dontxmit(sk);
390 		}
391 		if (tp->ecn_flags & TCP_ECN_DEMAND_CWR)
392 			th->ece = 1;
393 	}
394 }
395 
396 /* Constructs common control bits of non-data skb. If SYN/FIN is present,
397  * auto increment end seqno.
398  */
tcp_init_nondata_skb(struct sk_buff * skb,u32 seq,u8 flags)399 static void tcp_init_nondata_skb(struct sk_buff *skb, u32 seq, u8 flags)
400 {
401 	skb->ip_summed = CHECKSUM_PARTIAL;
402 
403 	TCP_SKB_CB(skb)->tcp_flags = flags;
404 
405 	tcp_skb_pcount_set(skb, 1);
406 
407 	TCP_SKB_CB(skb)->seq = seq;
408 	if (flags & (TCPHDR_SYN | TCPHDR_FIN))
409 		seq++;
410 	TCP_SKB_CB(skb)->end_seq = seq;
411 }
412 
tcp_urg_mode(const struct tcp_sock * tp)413 static inline bool tcp_urg_mode(const struct tcp_sock *tp)
414 {
415 	return tp->snd_una != tp->snd_up;
416 }
417 
418 #define OPTION_SACK_ADVERTISE	BIT(0)
419 #define OPTION_TS		BIT(1)
420 #define OPTION_MD5		BIT(2)
421 #define OPTION_WSCALE		BIT(3)
422 #define OPTION_FAST_OPEN_COOKIE	BIT(8)
423 #define OPTION_SMC		BIT(9)
424 #define OPTION_MPTCP		BIT(10)
425 
smc_options_write(__be32 * ptr,u16 * options)426 static void smc_options_write(__be32 *ptr, u16 *options)
427 {
428 #if IS_ENABLED(CONFIG_SMC)
429 	if (static_branch_unlikely(&tcp_have_smc)) {
430 		if (unlikely(OPTION_SMC & *options)) {
431 			*ptr++ = htonl((TCPOPT_NOP  << 24) |
432 				       (TCPOPT_NOP  << 16) |
433 				       (TCPOPT_EXP <<  8) |
434 				       (TCPOLEN_EXP_SMC_BASE));
435 			*ptr++ = htonl(TCPOPT_SMC_MAGIC);
436 		}
437 	}
438 #endif
439 }
440 
441 struct tcp_out_options {
442 	u16 options;		/* bit field of OPTION_* */
443 	u16 mss;		/* 0 to disable */
444 	u8 ws;			/* window scale, 0 to disable */
445 	u8 num_sack_blocks;	/* number of SACK blocks to include */
446 	u8 hash_size;		/* bytes in hash_location */
447 	u8 bpf_opt_len;		/* length of BPF hdr option */
448 	__u8 *hash_location;	/* temporary pointer, overloaded */
449 	__u32 tsval, tsecr;	/* need to include OPTION_TS */
450 	struct tcp_fastopen_cookie *fastopen_cookie;	/* Fast open cookie */
451 	struct mptcp_out_options mptcp;
452 };
453 
mptcp_options_write(struct tcphdr * th,__be32 * ptr,struct tcp_sock * tp,struct tcp_out_options * opts)454 static void mptcp_options_write(struct tcphdr *th, __be32 *ptr,
455 				struct tcp_sock *tp,
456 				struct tcp_out_options *opts)
457 {
458 #if IS_ENABLED(CONFIG_MPTCP)
459 	if (unlikely(OPTION_MPTCP & opts->options))
460 		mptcp_write_options(th, ptr, tp, &opts->mptcp);
461 #endif
462 }
463 
464 #ifdef CONFIG_CGROUP_BPF
bpf_skops_write_hdr_opt_arg0(struct sk_buff * skb,enum tcp_synack_type synack_type)465 static int bpf_skops_write_hdr_opt_arg0(struct sk_buff *skb,
466 					enum tcp_synack_type synack_type)
467 {
468 	if (unlikely(!skb))
469 		return BPF_WRITE_HDR_TCP_CURRENT_MSS;
470 
471 	if (unlikely(synack_type == TCP_SYNACK_COOKIE))
472 		return BPF_WRITE_HDR_TCP_SYNACK_COOKIE;
473 
474 	return 0;
475 }
476 
477 /* req, syn_skb and synack_type are used when writing synack */
bpf_skops_hdr_opt_len(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts,unsigned int * remaining)478 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
479 				  struct request_sock *req,
480 				  struct sk_buff *syn_skb,
481 				  enum tcp_synack_type synack_type,
482 				  struct tcp_out_options *opts,
483 				  unsigned int *remaining)
484 {
485 	struct bpf_sock_ops_kern sock_ops;
486 	int err;
487 
488 	if (likely(!BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk),
489 					   BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG)) ||
490 	    !*remaining)
491 		return;
492 
493 	/* *remaining has already been aligned to 4 bytes, so *remaining >= 4 */
494 
495 	/* init sock_ops */
496 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
497 
498 	sock_ops.op = BPF_SOCK_OPS_HDR_OPT_LEN_CB;
499 
500 	if (req) {
501 		/* The listen "sk" cannot be passed here because
502 		 * it is not locked.  It would not make too much
503 		 * sense to do bpf_setsockopt(listen_sk) based
504 		 * on individual connection request also.
505 		 *
506 		 * Thus, "req" is passed here and the cgroup-bpf-progs
507 		 * of the listen "sk" will be run.
508 		 *
509 		 * "req" is also used here for fastopen even the "sk" here is
510 		 * a fullsock "child" sk.  It is to keep the behavior
511 		 * consistent between fastopen and non-fastopen on
512 		 * the bpf programming side.
513 		 */
514 		sock_ops.sk = (struct sock *)req;
515 		sock_ops.syn_skb = syn_skb;
516 	} else {
517 		sock_owned_by_me(sk);
518 
519 		sock_ops.is_fullsock = 1;
520 		sock_ops.sk = sk;
521 	}
522 
523 	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
524 	sock_ops.remaining_opt_len = *remaining;
525 	/* tcp_current_mss() does not pass a skb */
526 	if (skb)
527 		bpf_skops_init_skb(&sock_ops, skb, 0);
528 
529 	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
530 
531 	if (err || sock_ops.remaining_opt_len == *remaining)
532 		return;
533 
534 	opts->bpf_opt_len = *remaining - sock_ops.remaining_opt_len;
535 	/* round up to 4 bytes */
536 	opts->bpf_opt_len = (opts->bpf_opt_len + 3) & ~3;
537 
538 	*remaining -= opts->bpf_opt_len;
539 }
540 
bpf_skops_write_hdr_opt(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts)541 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
542 				    struct request_sock *req,
543 				    struct sk_buff *syn_skb,
544 				    enum tcp_synack_type synack_type,
545 				    struct tcp_out_options *opts)
546 {
547 	u8 first_opt_off, nr_written, max_opt_len = opts->bpf_opt_len;
548 	struct bpf_sock_ops_kern sock_ops;
549 	int err;
550 
551 	if (likely(!max_opt_len))
552 		return;
553 
554 	memset(&sock_ops, 0, offsetof(struct bpf_sock_ops_kern, temp));
555 
556 	sock_ops.op = BPF_SOCK_OPS_WRITE_HDR_OPT_CB;
557 
558 	if (req) {
559 		sock_ops.sk = (struct sock *)req;
560 		sock_ops.syn_skb = syn_skb;
561 	} else {
562 		sock_owned_by_me(sk);
563 
564 		sock_ops.is_fullsock = 1;
565 		sock_ops.sk = sk;
566 	}
567 
568 	sock_ops.args[0] = bpf_skops_write_hdr_opt_arg0(skb, synack_type);
569 	sock_ops.remaining_opt_len = max_opt_len;
570 	first_opt_off = tcp_hdrlen(skb) - max_opt_len;
571 	bpf_skops_init_skb(&sock_ops, skb, first_opt_off);
572 
573 	err = BPF_CGROUP_RUN_PROG_SOCK_OPS_SK(&sock_ops, sk);
574 
575 	if (err)
576 		nr_written = 0;
577 	else
578 		nr_written = max_opt_len - sock_ops.remaining_opt_len;
579 
580 	if (nr_written < max_opt_len)
581 		memset(skb->data + first_opt_off + nr_written, TCPOPT_NOP,
582 		       max_opt_len - nr_written);
583 }
584 #else
bpf_skops_hdr_opt_len(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts,unsigned int * remaining)585 static void bpf_skops_hdr_opt_len(struct sock *sk, struct sk_buff *skb,
586 				  struct request_sock *req,
587 				  struct sk_buff *syn_skb,
588 				  enum tcp_synack_type synack_type,
589 				  struct tcp_out_options *opts,
590 				  unsigned int *remaining)
591 {
592 }
593 
bpf_skops_write_hdr_opt(struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct sk_buff * syn_skb,enum tcp_synack_type synack_type,struct tcp_out_options * opts)594 static void bpf_skops_write_hdr_opt(struct sock *sk, struct sk_buff *skb,
595 				    struct request_sock *req,
596 				    struct sk_buff *syn_skb,
597 				    enum tcp_synack_type synack_type,
598 				    struct tcp_out_options *opts)
599 {
600 }
601 #endif
602 
603 /* Write previously computed TCP options to the packet.
604  *
605  * Beware: Something in the Internet is very sensitive to the ordering of
606  * TCP options, we learned this through the hard way, so be careful here.
607  * Luckily we can at least blame others for their non-compliance but from
608  * inter-operability perspective it seems that we're somewhat stuck with
609  * the ordering which we have been using if we want to keep working with
610  * those broken things (not that it currently hurts anybody as there isn't
611  * particular reason why the ordering would need to be changed).
612  *
613  * At least SACK_PERM as the first option is known to lead to a disaster
614  * (but it may well be that other scenarios fail similarly).
615  */
tcp_options_write(struct tcphdr * th,struct tcp_sock * tp,struct tcp_out_options * opts)616 static void tcp_options_write(struct tcphdr *th, struct tcp_sock *tp,
617 			      struct tcp_out_options *opts)
618 {
619 	__be32 *ptr = (__be32 *)(th + 1);
620 	u16 options = opts->options;	/* mungable copy */
621 
622 	if (unlikely(OPTION_MD5 & options)) {
623 		*ptr++ = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
624 			       (TCPOPT_MD5SIG << 8) | TCPOLEN_MD5SIG);
625 		/* overload cookie hash location */
626 		opts->hash_location = (__u8 *)ptr;
627 		ptr += 4;
628 	}
629 
630 	if (unlikely(opts->mss)) {
631 		*ptr++ = htonl((TCPOPT_MSS << 24) |
632 			       (TCPOLEN_MSS << 16) |
633 			       opts->mss);
634 	}
635 
636 	if (likely(OPTION_TS & options)) {
637 		if (unlikely(OPTION_SACK_ADVERTISE & options)) {
638 			*ptr++ = htonl((TCPOPT_SACK_PERM << 24) |
639 				       (TCPOLEN_SACK_PERM << 16) |
640 				       (TCPOPT_TIMESTAMP << 8) |
641 				       TCPOLEN_TIMESTAMP);
642 			options &= ~OPTION_SACK_ADVERTISE;
643 		} else {
644 			*ptr++ = htonl((TCPOPT_NOP << 24) |
645 				       (TCPOPT_NOP << 16) |
646 				       (TCPOPT_TIMESTAMP << 8) |
647 				       TCPOLEN_TIMESTAMP);
648 		}
649 		*ptr++ = htonl(opts->tsval);
650 		*ptr++ = htonl(opts->tsecr);
651 	}
652 
653 	if (unlikely(OPTION_SACK_ADVERTISE & options)) {
654 		*ptr++ = htonl((TCPOPT_NOP << 24) |
655 			       (TCPOPT_NOP << 16) |
656 			       (TCPOPT_SACK_PERM << 8) |
657 			       TCPOLEN_SACK_PERM);
658 	}
659 
660 	if (unlikely(OPTION_WSCALE & options)) {
661 		*ptr++ = htonl((TCPOPT_NOP << 24) |
662 			       (TCPOPT_WINDOW << 16) |
663 			       (TCPOLEN_WINDOW << 8) |
664 			       opts->ws);
665 	}
666 
667 	if (unlikely(opts->num_sack_blocks)) {
668 		struct tcp_sack_block *sp = tp->rx_opt.dsack ?
669 			tp->duplicate_sack : tp->selective_acks;
670 		int this_sack;
671 
672 		*ptr++ = htonl((TCPOPT_NOP  << 24) |
673 			       (TCPOPT_NOP  << 16) |
674 			       (TCPOPT_SACK <<  8) |
675 			       (TCPOLEN_SACK_BASE + (opts->num_sack_blocks *
676 						     TCPOLEN_SACK_PERBLOCK)));
677 
678 		for (this_sack = 0; this_sack < opts->num_sack_blocks;
679 		     ++this_sack) {
680 			*ptr++ = htonl(sp[this_sack].start_seq);
681 			*ptr++ = htonl(sp[this_sack].end_seq);
682 		}
683 
684 		tp->rx_opt.dsack = 0;
685 	}
686 
687 	if (unlikely(OPTION_FAST_OPEN_COOKIE & options)) {
688 		struct tcp_fastopen_cookie *foc = opts->fastopen_cookie;
689 		u8 *p = (u8 *)ptr;
690 		u32 len; /* Fast Open option length */
691 
692 		if (foc->exp) {
693 			len = TCPOLEN_EXP_FASTOPEN_BASE + foc->len;
694 			*ptr = htonl((TCPOPT_EXP << 24) | (len << 16) |
695 				     TCPOPT_FASTOPEN_MAGIC);
696 			p += TCPOLEN_EXP_FASTOPEN_BASE;
697 		} else {
698 			len = TCPOLEN_FASTOPEN_BASE + foc->len;
699 			*p++ = TCPOPT_FASTOPEN;
700 			*p++ = len;
701 		}
702 
703 		memcpy(p, foc->val, foc->len);
704 		if ((len & 3) == 2) {
705 			p[foc->len] = TCPOPT_NOP;
706 			p[foc->len + 1] = TCPOPT_NOP;
707 		}
708 		ptr += (len + 3) >> 2;
709 	}
710 
711 	smc_options_write(ptr, &options);
712 
713 	mptcp_options_write(th, ptr, tp, opts);
714 }
715 
smc_set_option(const struct tcp_sock * tp,struct tcp_out_options * opts,unsigned int * remaining)716 static void smc_set_option(const struct tcp_sock *tp,
717 			   struct tcp_out_options *opts,
718 			   unsigned int *remaining)
719 {
720 #if IS_ENABLED(CONFIG_SMC)
721 	if (static_branch_unlikely(&tcp_have_smc)) {
722 		if (tp->syn_smc) {
723 			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
724 				opts->options |= OPTION_SMC;
725 				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
726 			}
727 		}
728 	}
729 #endif
730 }
731 
smc_set_option_cond(const struct tcp_sock * tp,const struct inet_request_sock * ireq,struct tcp_out_options * opts,unsigned int * remaining)732 static void smc_set_option_cond(const struct tcp_sock *tp,
733 				const struct inet_request_sock *ireq,
734 				struct tcp_out_options *opts,
735 				unsigned int *remaining)
736 {
737 #if IS_ENABLED(CONFIG_SMC)
738 	if (static_branch_unlikely(&tcp_have_smc)) {
739 		if (tp->syn_smc && ireq->smc_ok) {
740 			if (*remaining >= TCPOLEN_EXP_SMC_BASE_ALIGNED) {
741 				opts->options |= OPTION_SMC;
742 				*remaining -= TCPOLEN_EXP_SMC_BASE_ALIGNED;
743 			}
744 		}
745 	}
746 #endif
747 }
748 
mptcp_set_option_cond(const struct request_sock * req,struct tcp_out_options * opts,unsigned int * remaining)749 static void mptcp_set_option_cond(const struct request_sock *req,
750 				  struct tcp_out_options *opts,
751 				  unsigned int *remaining)
752 {
753 	if (rsk_is_mptcp(req)) {
754 		unsigned int size;
755 
756 		if (mptcp_synack_options(req, &size, &opts->mptcp)) {
757 			if (*remaining >= size) {
758 				opts->options |= OPTION_MPTCP;
759 				*remaining -= size;
760 			}
761 		}
762 	}
763 }
764 
765 /* Compute TCP options for SYN packets. This is not the final
766  * network wire format yet.
767  */
tcp_syn_options(struct sock * sk,struct sk_buff * skb,struct tcp_out_options * opts,struct tcp_md5sig_key ** md5)768 static unsigned int tcp_syn_options(struct sock *sk, struct sk_buff *skb,
769 				struct tcp_out_options *opts,
770 				struct tcp_md5sig_key **md5)
771 {
772 	struct tcp_sock *tp = tcp_sk(sk);
773 	unsigned int remaining = MAX_TCP_OPTION_SPACE;
774 	struct tcp_fastopen_request *fastopen = tp->fastopen_req;
775 
776 	*md5 = NULL;
777 #ifdef CONFIG_TCP_MD5SIG
778 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
779 	    rcu_access_pointer(tp->md5sig_info)) {
780 		*md5 = tp->af_specific->md5_lookup(sk, sk);
781 		if (*md5) {
782 			opts->options |= OPTION_MD5;
783 			remaining -= TCPOLEN_MD5SIG_ALIGNED;
784 		}
785 	}
786 #endif
787 
788 	/* We always get an MSS option.  The option bytes which will be seen in
789 	 * normal data packets should timestamps be used, must be in the MSS
790 	 * advertised.  But we subtract them from tp->mss_cache so that
791 	 * calculations in tcp_sendmsg are simpler etc.  So account for this
792 	 * fact here if necessary.  If we don't do this correctly, as a
793 	 * receiver we won't recognize data packets as being full sized when we
794 	 * should, and thus we won't abide by the delayed ACK rules correctly.
795 	 * SACKs don't matter, we never delay an ACK when we have any of those
796 	 * going out.  */
797 	opts->mss = tcp_advertise_mss(sk);
798 	remaining -= TCPOLEN_MSS_ALIGNED;
799 
800 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps) && !*md5)) {
801 		opts->options |= OPTION_TS;
802 		opts->tsval = tcp_skb_timestamp(skb) + tp->tsoffset;
803 		opts->tsecr = tp->rx_opt.ts_recent;
804 		remaining -= TCPOLEN_TSTAMP_ALIGNED;
805 	}
806 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling))) {
807 		opts->ws = tp->rx_opt.rcv_wscale;
808 		opts->options |= OPTION_WSCALE;
809 		remaining -= TCPOLEN_WSCALE_ALIGNED;
810 	}
811 	if (likely(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_sack))) {
812 		opts->options |= OPTION_SACK_ADVERTISE;
813 		if (unlikely(!(OPTION_TS & opts->options)))
814 			remaining -= TCPOLEN_SACKPERM_ALIGNED;
815 	}
816 
817 	if (fastopen && fastopen->cookie.len >= 0) {
818 		u32 need = fastopen->cookie.len;
819 
820 		need += fastopen->cookie.exp ? TCPOLEN_EXP_FASTOPEN_BASE :
821 					       TCPOLEN_FASTOPEN_BASE;
822 		need = (need + 3) & ~3U;  /* Align to 32 bits */
823 		if (remaining >= need) {
824 			opts->options |= OPTION_FAST_OPEN_COOKIE;
825 			opts->fastopen_cookie = &fastopen->cookie;
826 			remaining -= need;
827 			tp->syn_fastopen = 1;
828 			tp->syn_fastopen_exp = fastopen->cookie.exp ? 1 : 0;
829 		}
830 	}
831 
832 	smc_set_option(tp, opts, &remaining);
833 
834 	if (sk_is_mptcp(sk)) {
835 		unsigned int size;
836 
837 		if (mptcp_syn_options(sk, skb, &size, &opts->mptcp)) {
838 			opts->options |= OPTION_MPTCP;
839 			remaining -= size;
840 		}
841 	}
842 
843 	bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
844 
845 	return MAX_TCP_OPTION_SPACE - remaining;
846 }
847 
848 /* Set up TCP options for SYN-ACKs. */
tcp_synack_options(const struct sock * sk,struct request_sock * req,unsigned int mss,struct sk_buff * skb,struct tcp_out_options * opts,const struct tcp_md5sig_key * md5,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)849 static unsigned int tcp_synack_options(const struct sock *sk,
850 				       struct request_sock *req,
851 				       unsigned int mss, struct sk_buff *skb,
852 				       struct tcp_out_options *opts,
853 				       const struct tcp_md5sig_key *md5,
854 				       struct tcp_fastopen_cookie *foc,
855 				       enum tcp_synack_type synack_type,
856 				       struct sk_buff *syn_skb)
857 {
858 	struct inet_request_sock *ireq = inet_rsk(req);
859 	unsigned int remaining = MAX_TCP_OPTION_SPACE;
860 
861 #ifdef CONFIG_TCP_MD5SIG
862 	if (md5) {
863 		opts->options |= OPTION_MD5;
864 		remaining -= TCPOLEN_MD5SIG_ALIGNED;
865 
866 		/* We can't fit any SACK blocks in a packet with MD5 + TS
867 		 * options. There was discussion about disabling SACK
868 		 * rather than TS in order to fit in better with old,
869 		 * buggy kernels, but that was deemed to be unnecessary.
870 		 */
871 		if (synack_type != TCP_SYNACK_COOKIE)
872 			ireq->tstamp_ok &= !ireq->sack_ok;
873 	}
874 #endif
875 
876 	/* We always send an MSS option. */
877 	opts->mss = mss;
878 	remaining -= TCPOLEN_MSS_ALIGNED;
879 
880 	if (likely(ireq->wscale_ok)) {
881 		opts->ws = ireq->rcv_wscale;
882 		opts->options |= OPTION_WSCALE;
883 		remaining -= TCPOLEN_WSCALE_ALIGNED;
884 	}
885 	if (likely(ireq->tstamp_ok)) {
886 		opts->options |= OPTION_TS;
887 		opts->tsval = tcp_skb_timestamp(skb) + tcp_rsk(req)->ts_off;
888 		opts->tsecr = READ_ONCE(req->ts_recent);
889 		remaining -= TCPOLEN_TSTAMP_ALIGNED;
890 	}
891 	if (likely(ireq->sack_ok)) {
892 		opts->options |= OPTION_SACK_ADVERTISE;
893 		if (unlikely(!ireq->tstamp_ok))
894 			remaining -= TCPOLEN_SACKPERM_ALIGNED;
895 	}
896 	if (foc != NULL && foc->len >= 0) {
897 		u32 need = foc->len;
898 
899 		need += foc->exp ? TCPOLEN_EXP_FASTOPEN_BASE :
900 				   TCPOLEN_FASTOPEN_BASE;
901 		need = (need + 3) & ~3U;  /* Align to 32 bits */
902 		if (remaining >= need) {
903 			opts->options |= OPTION_FAST_OPEN_COOKIE;
904 			opts->fastopen_cookie = foc;
905 			remaining -= need;
906 		}
907 	}
908 
909 	mptcp_set_option_cond(req, opts, &remaining);
910 
911 	smc_set_option_cond(tcp_sk(sk), ireq, opts, &remaining);
912 
913 	bpf_skops_hdr_opt_len((struct sock *)sk, skb, req, syn_skb,
914 			      synack_type, opts, &remaining);
915 
916 	return MAX_TCP_OPTION_SPACE - remaining;
917 }
918 
919 /* Compute TCP options for ESTABLISHED sockets. This is not the
920  * final wire format yet.
921  */
tcp_established_options(struct sock * sk,struct sk_buff * skb,struct tcp_out_options * opts,struct tcp_md5sig_key ** md5)922 static unsigned int tcp_established_options(struct sock *sk, struct sk_buff *skb,
923 					struct tcp_out_options *opts,
924 					struct tcp_md5sig_key **md5)
925 {
926 	struct tcp_sock *tp = tcp_sk(sk);
927 	unsigned int size = 0;
928 	unsigned int eff_sacks;
929 
930 	opts->options = 0;
931 
932 	*md5 = NULL;
933 #ifdef CONFIG_TCP_MD5SIG
934 	if (static_branch_unlikely(&tcp_md5_needed.key) &&
935 	    rcu_access_pointer(tp->md5sig_info)) {
936 		*md5 = tp->af_specific->md5_lookup(sk, sk);
937 		if (*md5) {
938 			opts->options |= OPTION_MD5;
939 			size += TCPOLEN_MD5SIG_ALIGNED;
940 		}
941 	}
942 #endif
943 
944 	if (likely(tp->rx_opt.tstamp_ok)) {
945 		opts->options |= OPTION_TS;
946 		opts->tsval = skb ? tcp_skb_timestamp(skb) + tp->tsoffset : 0;
947 		opts->tsecr = tp->rx_opt.ts_recent;
948 		size += TCPOLEN_TSTAMP_ALIGNED;
949 	}
950 
951 	/* MPTCP options have precedence over SACK for the limited TCP
952 	 * option space because a MPTCP connection would be forced to
953 	 * fall back to regular TCP if a required multipath option is
954 	 * missing. SACK still gets a chance to use whatever space is
955 	 * left.
956 	 */
957 	if (sk_is_mptcp(sk)) {
958 		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
959 		unsigned int opt_size = 0;
960 
961 		if (mptcp_established_options(sk, skb, &opt_size, remaining,
962 					      &opts->mptcp)) {
963 			opts->options |= OPTION_MPTCP;
964 			size += opt_size;
965 		}
966 	}
967 
968 	eff_sacks = tp->rx_opt.num_sacks + tp->rx_opt.dsack;
969 	if (unlikely(eff_sacks)) {
970 		const unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
971 		if (unlikely(remaining < TCPOLEN_SACK_BASE_ALIGNED +
972 					 TCPOLEN_SACK_PERBLOCK))
973 			return size;
974 
975 		opts->num_sack_blocks =
976 			min_t(unsigned int, eff_sacks,
977 			      (remaining - TCPOLEN_SACK_BASE_ALIGNED) /
978 			      TCPOLEN_SACK_PERBLOCK);
979 
980 		size += TCPOLEN_SACK_BASE_ALIGNED +
981 			opts->num_sack_blocks * TCPOLEN_SACK_PERBLOCK;
982 	}
983 
984 	if (unlikely(BPF_SOCK_OPS_TEST_FLAG(tp,
985 					    BPF_SOCK_OPS_WRITE_HDR_OPT_CB_FLAG))) {
986 		unsigned int remaining = MAX_TCP_OPTION_SPACE - size;
987 
988 		bpf_skops_hdr_opt_len(sk, skb, NULL, NULL, 0, opts, &remaining);
989 
990 		size = MAX_TCP_OPTION_SPACE - remaining;
991 	}
992 
993 	return size;
994 }
995 
996 
997 /* TCP SMALL QUEUES (TSQ)
998  *
999  * TSQ goal is to keep small amount of skbs per tcp flow in tx queues (qdisc+dev)
1000  * to reduce RTT and bufferbloat.
1001  * We do this using a special skb destructor (tcp_wfree).
1002  *
1003  * Its important tcp_wfree() can be replaced by sock_wfree() in the event skb
1004  * needs to be reallocated in a driver.
1005  * The invariant being skb->truesize subtracted from sk->sk_wmem_alloc
1006  *
1007  * Since transmit from skb destructor is forbidden, we use a tasklet
1008  * to process all sockets that eventually need to send more skbs.
1009  * We use one tasklet per cpu, with its own queue of sockets.
1010  */
1011 struct tsq_tasklet {
1012 	struct tasklet_struct	tasklet;
1013 	struct list_head	head; /* queue of tcp sockets */
1014 };
1015 static DEFINE_PER_CPU(struct tsq_tasklet, tsq_tasklet);
1016 
tcp_tsq_write(struct sock * sk)1017 static void tcp_tsq_write(struct sock *sk)
1018 {
1019 	if ((1 << sk->sk_state) &
1020 	    (TCPF_ESTABLISHED | TCPF_FIN_WAIT1 | TCPF_CLOSING |
1021 	     TCPF_CLOSE_WAIT  | TCPF_LAST_ACK)) {
1022 		struct tcp_sock *tp = tcp_sk(sk);
1023 
1024 		if (tp->lost_out > tp->retrans_out &&
1025 		    tcp_snd_cwnd(tp) > tcp_packets_in_flight(tp)) {
1026 			tcp_mstamp_refresh(tp);
1027 			tcp_xmit_retransmit_queue(sk);
1028 		}
1029 
1030 		tcp_write_xmit(sk, tcp_current_mss(sk), tp->nonagle,
1031 			       0, GFP_ATOMIC);
1032 	}
1033 }
1034 
tcp_tsq_handler(struct sock * sk)1035 static void tcp_tsq_handler(struct sock *sk)
1036 {
1037 	bh_lock_sock(sk);
1038 	if (!sock_owned_by_user(sk))
1039 		tcp_tsq_write(sk);
1040 	else if (!test_and_set_bit(TCP_TSQ_DEFERRED, &sk->sk_tsq_flags))
1041 		sock_hold(sk);
1042 	bh_unlock_sock(sk);
1043 }
1044 /*
1045  * One tasklet per cpu tries to send more skbs.
1046  * We run in tasklet context but need to disable irqs when
1047  * transferring tsq->head because tcp_wfree() might
1048  * interrupt us (non NAPI drivers)
1049  */
tcp_tasklet_func(struct tasklet_struct * t)1050 static void tcp_tasklet_func(struct tasklet_struct *t)
1051 {
1052 	struct tsq_tasklet *tsq = from_tasklet(tsq,  t, tasklet);
1053 	LIST_HEAD(list);
1054 	unsigned long flags;
1055 	struct list_head *q, *n;
1056 	struct tcp_sock *tp;
1057 	struct sock *sk;
1058 
1059 	local_irq_save(flags);
1060 	list_splice_init(&tsq->head, &list);
1061 	local_irq_restore(flags);
1062 
1063 	list_for_each_safe(q, n, &list) {
1064 		tp = list_entry(q, struct tcp_sock, tsq_node);
1065 		list_del(&tp->tsq_node);
1066 
1067 		sk = (struct sock *)tp;
1068 		smp_mb__before_atomic();
1069 		clear_bit(TSQ_QUEUED, &sk->sk_tsq_flags);
1070 
1071 		tcp_tsq_handler(sk);
1072 		sk_free(sk);
1073 	}
1074 }
1075 
1076 #define TCP_DEFERRED_ALL (TCPF_TSQ_DEFERRED |		\
1077 			  TCPF_WRITE_TIMER_DEFERRED |	\
1078 			  TCPF_DELACK_TIMER_DEFERRED |	\
1079 			  TCPF_MTU_REDUCED_DEFERRED)
1080 /**
1081  * tcp_release_cb - tcp release_sock() callback
1082  * @sk: socket
1083  *
1084  * called from release_sock() to perform protocol dependent
1085  * actions before socket release.
1086  */
tcp_release_cb(struct sock * sk)1087 void tcp_release_cb(struct sock *sk)
1088 {
1089 	unsigned long flags = smp_load_acquire(&sk->sk_tsq_flags);
1090 	unsigned long nflags;
1091 
1092 	/* perform an atomic operation only if at least one flag is set */
1093 	do {
1094 		if (!(flags & TCP_DEFERRED_ALL))
1095 			return;
1096 		nflags = flags & ~TCP_DEFERRED_ALL;
1097 	} while (!try_cmpxchg(&sk->sk_tsq_flags, &flags, nflags));
1098 
1099 	if (flags & TCPF_TSQ_DEFERRED) {
1100 		tcp_tsq_write(sk);
1101 		__sock_put(sk);
1102 	}
1103 	/* Here begins the tricky part :
1104 	 * We are called from release_sock() with :
1105 	 * 1) BH disabled
1106 	 * 2) sk_lock.slock spinlock held
1107 	 * 3) socket owned by us (sk->sk_lock.owned == 1)
1108 	 *
1109 	 * But following code is meant to be called from BH handlers,
1110 	 * so we should keep BH disabled, but early release socket ownership
1111 	 */
1112 	sock_release_ownership(sk);
1113 
1114 	if (flags & TCPF_WRITE_TIMER_DEFERRED) {
1115 		tcp_write_timer_handler(sk);
1116 		__sock_put(sk);
1117 	}
1118 	if (flags & TCPF_DELACK_TIMER_DEFERRED) {
1119 		tcp_delack_timer_handler(sk);
1120 		__sock_put(sk);
1121 	}
1122 	if (flags & TCPF_MTU_REDUCED_DEFERRED) {
1123 		inet_csk(sk)->icsk_af_ops->mtu_reduced(sk);
1124 		__sock_put(sk);
1125 	}
1126 }
1127 EXPORT_SYMBOL(tcp_release_cb);
1128 
tcp_tasklet_init(void)1129 void __init tcp_tasklet_init(void)
1130 {
1131 	int i;
1132 
1133 	for_each_possible_cpu(i) {
1134 		struct tsq_tasklet *tsq = &per_cpu(tsq_tasklet, i);
1135 
1136 		INIT_LIST_HEAD(&tsq->head);
1137 		tasklet_setup(&tsq->tasklet, tcp_tasklet_func);
1138 	}
1139 }
1140 
1141 /*
1142  * Write buffer destructor automatically called from kfree_skb.
1143  * We can't xmit new skbs from this context, as we might already
1144  * hold qdisc lock.
1145  */
tcp_wfree(struct sk_buff * skb)1146 void tcp_wfree(struct sk_buff *skb)
1147 {
1148 	struct sock *sk = skb->sk;
1149 	struct tcp_sock *tp = tcp_sk(sk);
1150 	unsigned long flags, nval, oval;
1151 	struct tsq_tasklet *tsq;
1152 	bool empty;
1153 
1154 	/* Keep one reference on sk_wmem_alloc.
1155 	 * Will be released by sk_free() from here or tcp_tasklet_func()
1156 	 */
1157 	WARN_ON(refcount_sub_and_test(skb->truesize - 1, &sk->sk_wmem_alloc));
1158 
1159 	/* If this softirq is serviced by ksoftirqd, we are likely under stress.
1160 	 * Wait until our queues (qdisc + devices) are drained.
1161 	 * This gives :
1162 	 * - less callbacks to tcp_write_xmit(), reducing stress (batches)
1163 	 * - chance for incoming ACK (processed by another cpu maybe)
1164 	 *   to migrate this flow (skb->ooo_okay will be eventually set)
1165 	 */
1166 	if (refcount_read(&sk->sk_wmem_alloc) >= SKB_TRUESIZE(1) && this_cpu_ksoftirqd() == current)
1167 		goto out;
1168 
1169 	oval = smp_load_acquire(&sk->sk_tsq_flags);
1170 	do {
1171 		if (!(oval & TSQF_THROTTLED) || (oval & TSQF_QUEUED))
1172 			goto out;
1173 
1174 		nval = (oval & ~TSQF_THROTTLED) | TSQF_QUEUED;
1175 	} while (!try_cmpxchg(&sk->sk_tsq_flags, &oval, nval));
1176 
1177 	/* queue this socket to tasklet queue */
1178 	local_irq_save(flags);
1179 	tsq = this_cpu_ptr(&tsq_tasklet);
1180 	empty = list_empty(&tsq->head);
1181 	list_add(&tp->tsq_node, &tsq->head);
1182 	if (empty)
1183 		tasklet_schedule(&tsq->tasklet);
1184 	local_irq_restore(flags);
1185 	return;
1186 out:
1187 	sk_free(sk);
1188 }
1189 
1190 /* Note: Called under soft irq.
1191  * We can call TCP stack right away, unless socket is owned by user.
1192  */
tcp_pace_kick(struct hrtimer * timer)1193 enum hrtimer_restart tcp_pace_kick(struct hrtimer *timer)
1194 {
1195 	struct tcp_sock *tp = container_of(timer, struct tcp_sock, pacing_timer);
1196 	struct sock *sk = (struct sock *)tp;
1197 
1198 	tcp_tsq_handler(sk);
1199 	sock_put(sk);
1200 
1201 	return HRTIMER_NORESTART;
1202 }
1203 
tcp_update_skb_after_send(struct sock * sk,struct sk_buff * skb,u64 prior_wstamp)1204 static void tcp_update_skb_after_send(struct sock *sk, struct sk_buff *skb,
1205 				      u64 prior_wstamp)
1206 {
1207 	struct tcp_sock *tp = tcp_sk(sk);
1208 
1209 	if (sk->sk_pacing_status != SK_PACING_NONE) {
1210 		unsigned long rate = sk->sk_pacing_rate;
1211 
1212 		/* Original sch_fq does not pace first 10 MSS
1213 		 * Note that tp->data_segs_out overflows after 2^32 packets,
1214 		 * this is a minor annoyance.
1215 		 */
1216 		if (rate != ~0UL && rate && tp->data_segs_out >= 10) {
1217 			u64 len_ns = div64_ul((u64)skb->len * NSEC_PER_SEC, rate);
1218 			u64 credit = tp->tcp_wstamp_ns - prior_wstamp;
1219 
1220 			/* take into account OS jitter */
1221 			len_ns -= min_t(u64, len_ns / 2, credit);
1222 			tp->tcp_wstamp_ns += len_ns;
1223 		}
1224 	}
1225 	list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
1226 }
1227 
1228 INDIRECT_CALLABLE_DECLARE(int ip_queue_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1229 INDIRECT_CALLABLE_DECLARE(int inet6_csk_xmit(struct sock *sk, struct sk_buff *skb, struct flowi *fl));
1230 INDIRECT_CALLABLE_DECLARE(void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb));
1231 
1232 /* This routine actually transmits TCP packets queued in by
1233  * tcp_do_sendmsg().  This is used by both the initial
1234  * transmission and possible later retransmissions.
1235  * All SKB's seen here are completely headerless.  It is our
1236  * job to build the TCP header, and pass the packet down to
1237  * IP so it can do the same plus pass the packet off to the
1238  * device.
1239  *
1240  * We are working here with either a clone of the original
1241  * SKB, or a fresh unique copy made by the retransmit engine.
1242  */
__tcp_transmit_skb(struct sock * sk,struct sk_buff * skb,int clone_it,gfp_t gfp_mask,u32 rcv_nxt)1243 static int __tcp_transmit_skb(struct sock *sk, struct sk_buff *skb,
1244 			      int clone_it, gfp_t gfp_mask, u32 rcv_nxt)
1245 {
1246 	const struct inet_connection_sock *icsk = inet_csk(sk);
1247 	struct inet_sock *inet;
1248 	struct tcp_sock *tp;
1249 	struct tcp_skb_cb *tcb;
1250 	struct tcp_out_options opts;
1251 	unsigned int tcp_options_size, tcp_header_size;
1252 	struct sk_buff *oskb = NULL;
1253 	struct tcp_md5sig_key *md5;
1254 	struct tcphdr *th;
1255 	u64 prior_wstamp;
1256 	int err;
1257 
1258 	BUG_ON(!skb || !tcp_skb_pcount(skb));
1259 	tp = tcp_sk(sk);
1260 	prior_wstamp = tp->tcp_wstamp_ns;
1261 	tp->tcp_wstamp_ns = max(tp->tcp_wstamp_ns, tp->tcp_clock_cache);
1262 	skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true);
1263 	if (clone_it) {
1264 		oskb = skb;
1265 
1266 		tcp_skb_tsorted_save(oskb) {
1267 			if (unlikely(skb_cloned(oskb)))
1268 				skb = pskb_copy(oskb, gfp_mask);
1269 			else
1270 				skb = skb_clone(oskb, gfp_mask);
1271 		} tcp_skb_tsorted_restore(oskb);
1272 
1273 		if (unlikely(!skb))
1274 			return -ENOBUFS;
1275 		/* retransmit skbs might have a non zero value in skb->dev
1276 		 * because skb->dev is aliased with skb->rbnode.rb_left
1277 		 */
1278 		skb->dev = NULL;
1279 	}
1280 
1281 	inet = inet_sk(sk);
1282 	tcb = TCP_SKB_CB(skb);
1283 	memset(&opts, 0, sizeof(opts));
1284 
1285 	if (unlikely(tcb->tcp_flags & TCPHDR_SYN)) {
1286 		tcp_options_size = tcp_syn_options(sk, skb, &opts, &md5);
1287 	} else {
1288 		tcp_options_size = tcp_established_options(sk, skb, &opts,
1289 							   &md5);
1290 		/* Force a PSH flag on all (GSO) packets to expedite GRO flush
1291 		 * at receiver : This slightly improve GRO performance.
1292 		 * Note that we do not force the PSH flag for non GSO packets,
1293 		 * because they might be sent under high congestion events,
1294 		 * and in this case it is better to delay the delivery of 1-MSS
1295 		 * packets and thus the corresponding ACK packet that would
1296 		 * release the following packet.
1297 		 */
1298 		if (tcp_skb_pcount(skb) > 1)
1299 			tcb->tcp_flags |= TCPHDR_PSH;
1300 	}
1301 	tcp_header_size = tcp_options_size + sizeof(struct tcphdr);
1302 
1303 	/* We set skb->ooo_okay to one if this packet can select
1304 	 * a different TX queue than prior packets of this flow,
1305 	 * to avoid self inflicted reorders.
1306 	 * The 'other' queue decision is based on current cpu number
1307 	 * if XPS is enabled, or sk->sk_txhash otherwise.
1308 	 * We can switch to another (and better) queue if:
1309 	 * 1) No packet with payload is in qdisc/device queues.
1310 	 *    Delays in TX completion can defeat the test
1311 	 *    even if packets were already sent.
1312 	 * 2) Or rtx queue is empty.
1313 	 *    This mitigates above case if ACK packets for
1314 	 *    all prior packets were already processed.
1315 	 */
1316 	skb->ooo_okay = sk_wmem_alloc_get(sk) < SKB_TRUESIZE(1) ||
1317 			tcp_rtx_queue_empty(sk);
1318 
1319 	/* If we had to use memory reserve to allocate this skb,
1320 	 * this might cause drops if packet is looped back :
1321 	 * Other socket might not have SOCK_MEMALLOC.
1322 	 * Packets not looped back do not care about pfmemalloc.
1323 	 */
1324 	skb->pfmemalloc = 0;
1325 
1326 	skb_push(skb, tcp_header_size);
1327 	skb_reset_transport_header(skb);
1328 
1329 	skb_orphan(skb);
1330 	skb->sk = sk;
1331 	skb->destructor = skb_is_tcp_pure_ack(skb) ? __sock_wfree : tcp_wfree;
1332 	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
1333 
1334 	skb_set_dst_pending_confirm(skb, READ_ONCE(sk->sk_dst_pending_confirm));
1335 
1336 	/* Build TCP header and checksum it. */
1337 	th = (struct tcphdr *)skb->data;
1338 	th->source		= inet->inet_sport;
1339 	th->dest		= inet->inet_dport;
1340 	th->seq			= htonl(tcb->seq);
1341 	th->ack_seq		= htonl(rcv_nxt);
1342 	*(((__be16 *)th) + 6)	= htons(((tcp_header_size >> 2) << 12) |
1343 					tcb->tcp_flags);
1344 
1345 	th->check		= 0;
1346 	th->urg_ptr		= 0;
1347 
1348 	/* The urg_mode check is necessary during a below snd_una win probe */
1349 	if (unlikely(tcp_urg_mode(tp) && before(tcb->seq, tp->snd_up))) {
1350 		if (before(tp->snd_up, tcb->seq + 0x10000)) {
1351 			th->urg_ptr = htons(tp->snd_up - tcb->seq);
1352 			th->urg = 1;
1353 		} else if (after(tcb->seq + 0xFFFF, tp->snd_nxt)) {
1354 			th->urg_ptr = htons(0xFFFF);
1355 			th->urg = 1;
1356 		}
1357 	}
1358 
1359 	skb_shinfo(skb)->gso_type = sk->sk_gso_type;
1360 	if (likely(!(tcb->tcp_flags & TCPHDR_SYN))) {
1361 		th->window      = htons(tcp_select_window(sk));
1362 		tcp_ecn_send(sk, skb, th, tcp_header_size);
1363 	} else {
1364 		/* RFC1323: The window in SYN & SYN/ACK segments
1365 		 * is never scaled.
1366 		 */
1367 		th->window	= htons(min(tp->rcv_wnd, 65535U));
1368 	}
1369 
1370 	tcp_options_write(th, tp, &opts);
1371 
1372 #ifdef CONFIG_TCP_MD5SIG
1373 	/* Calculate the MD5 hash, as we have all we need now */
1374 	if (md5) {
1375 		sk_gso_disable(sk);
1376 		tp->af_specific->calc_md5_hash(opts.hash_location,
1377 					       md5, sk, skb);
1378 	}
1379 #endif
1380 
1381 	/* BPF prog is the last one writing header option */
1382 	bpf_skops_write_hdr_opt(sk, skb, NULL, NULL, 0, &opts);
1383 
1384 	INDIRECT_CALL_INET(icsk->icsk_af_ops->send_check,
1385 			   tcp_v6_send_check, tcp_v4_send_check,
1386 			   sk, skb);
1387 
1388 	if (likely(tcb->tcp_flags & TCPHDR_ACK))
1389 		tcp_event_ack_sent(sk, rcv_nxt);
1390 
1391 	if (skb->len != tcp_header_size) {
1392 		tcp_event_data_sent(tp, sk);
1393 		tp->data_segs_out += tcp_skb_pcount(skb);
1394 		tp->bytes_sent += skb->len - tcp_header_size;
1395 	}
1396 
1397 	if (after(tcb->end_seq, tp->snd_nxt) || tcb->seq == tcb->end_seq)
1398 		TCP_ADD_STATS(sock_net(sk), TCP_MIB_OUTSEGS,
1399 			      tcp_skb_pcount(skb));
1400 
1401 	tp->segs_out += tcp_skb_pcount(skb);
1402 	skb_set_hash_from_sk(skb, sk);
1403 	/* OK, its time to fill skb_shinfo(skb)->gso_{segs|size} */
1404 	skb_shinfo(skb)->gso_segs = tcp_skb_pcount(skb);
1405 	skb_shinfo(skb)->gso_size = tcp_skb_mss(skb);
1406 
1407 	/* Leave earliest departure time in skb->tstamp (skb->skb_mstamp_ns) */
1408 
1409 	/* Cleanup our debris for IP stacks */
1410 	memset(skb->cb, 0, max(sizeof(struct inet_skb_parm),
1411 			       sizeof(struct inet6_skb_parm)));
1412 
1413 	tcp_add_tx_delay(skb, tp);
1414 
1415 	err = INDIRECT_CALL_INET(icsk->icsk_af_ops->queue_xmit,
1416 				 inet6_csk_xmit, ip_queue_xmit,
1417 				 sk, skb, &inet->cork.fl);
1418 
1419 	if (unlikely(err > 0)) {
1420 		tcp_enter_cwr(sk);
1421 		err = net_xmit_eval(err);
1422 	}
1423 	if (!err && oskb) {
1424 		tcp_update_skb_after_send(sk, oskb, prior_wstamp);
1425 		tcp_rate_skb_sent(sk, oskb);
1426 	}
1427 	return err;
1428 }
1429 
tcp_transmit_skb(struct sock * sk,struct sk_buff * skb,int clone_it,gfp_t gfp_mask)1430 static int tcp_transmit_skb(struct sock *sk, struct sk_buff *skb, int clone_it,
1431 			    gfp_t gfp_mask)
1432 {
1433 	return __tcp_transmit_skb(sk, skb, clone_it, gfp_mask,
1434 				  tcp_sk(sk)->rcv_nxt);
1435 }
1436 
1437 /* This routine just queues the buffer for sending.
1438  *
1439  * NOTE: probe0 timer is not checked, do not forget tcp_push_pending_frames,
1440  * otherwise socket can stall.
1441  */
tcp_queue_skb(struct sock * sk,struct sk_buff * skb)1442 static void tcp_queue_skb(struct sock *sk, struct sk_buff *skb)
1443 {
1444 	struct tcp_sock *tp = tcp_sk(sk);
1445 
1446 	/* Advance write_seq and place onto the write_queue. */
1447 	WRITE_ONCE(tp->write_seq, TCP_SKB_CB(skb)->end_seq);
1448 	__skb_header_release(skb);
1449 	tcp_add_write_queue_tail(sk, skb);
1450 	sk_wmem_queued_add(sk, skb->truesize);
1451 	sk_mem_charge(sk, skb->truesize);
1452 }
1453 
1454 /* Initialize TSO segments for a packet. */
tcp_set_skb_tso_segs(struct sk_buff * skb,unsigned int mss_now)1455 static void tcp_set_skb_tso_segs(struct sk_buff *skb, unsigned int mss_now)
1456 {
1457 	if (skb->len <= mss_now) {
1458 		/* Avoid the costly divide in the normal
1459 		 * non-TSO case.
1460 		 */
1461 		tcp_skb_pcount_set(skb, 1);
1462 		TCP_SKB_CB(skb)->tcp_gso_size = 0;
1463 	} else {
1464 		tcp_skb_pcount_set(skb, DIV_ROUND_UP(skb->len, mss_now));
1465 		TCP_SKB_CB(skb)->tcp_gso_size = mss_now;
1466 	}
1467 }
1468 
1469 /* Pcount in the middle of the write queue got changed, we need to do various
1470  * tweaks to fix counters
1471  */
tcp_adjust_pcount(struct sock * sk,const struct sk_buff * skb,int decr)1472 static void tcp_adjust_pcount(struct sock *sk, const struct sk_buff *skb, int decr)
1473 {
1474 	struct tcp_sock *tp = tcp_sk(sk);
1475 
1476 	tp->packets_out -= decr;
1477 
1478 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
1479 		tp->sacked_out -= decr;
1480 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS)
1481 		tp->retrans_out -= decr;
1482 	if (TCP_SKB_CB(skb)->sacked & TCPCB_LOST)
1483 		tp->lost_out -= decr;
1484 
1485 	/* Reno case is special. Sigh... */
1486 	if (tcp_is_reno(tp) && decr > 0)
1487 		tp->sacked_out -= min_t(u32, tp->sacked_out, decr);
1488 
1489 	if (tp->lost_skb_hint &&
1490 	    before(TCP_SKB_CB(skb)->seq, TCP_SKB_CB(tp->lost_skb_hint)->seq) &&
1491 	    (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED))
1492 		tp->lost_cnt_hint -= decr;
1493 
1494 	tcp_verify_left_out(tp);
1495 }
1496 
tcp_has_tx_tstamp(const struct sk_buff * skb)1497 static bool tcp_has_tx_tstamp(const struct sk_buff *skb)
1498 {
1499 	return TCP_SKB_CB(skb)->txstamp_ack ||
1500 		(skb_shinfo(skb)->tx_flags & SKBTX_ANY_TSTAMP);
1501 }
1502 
tcp_fragment_tstamp(struct sk_buff * skb,struct sk_buff * skb2)1503 static void tcp_fragment_tstamp(struct sk_buff *skb, struct sk_buff *skb2)
1504 {
1505 	struct skb_shared_info *shinfo = skb_shinfo(skb);
1506 
1507 	if (unlikely(tcp_has_tx_tstamp(skb)) &&
1508 	    !before(shinfo->tskey, TCP_SKB_CB(skb2)->seq)) {
1509 		struct skb_shared_info *shinfo2 = skb_shinfo(skb2);
1510 		u8 tsflags = shinfo->tx_flags & SKBTX_ANY_TSTAMP;
1511 
1512 		shinfo->tx_flags &= ~tsflags;
1513 		shinfo2->tx_flags |= tsflags;
1514 		swap(shinfo->tskey, shinfo2->tskey);
1515 		TCP_SKB_CB(skb2)->txstamp_ack = TCP_SKB_CB(skb)->txstamp_ack;
1516 		TCP_SKB_CB(skb)->txstamp_ack = 0;
1517 	}
1518 }
1519 
tcp_skb_fragment_eor(struct sk_buff * skb,struct sk_buff * skb2)1520 static void tcp_skb_fragment_eor(struct sk_buff *skb, struct sk_buff *skb2)
1521 {
1522 	TCP_SKB_CB(skb2)->eor = TCP_SKB_CB(skb)->eor;
1523 	TCP_SKB_CB(skb)->eor = 0;
1524 }
1525 
1526 /* Insert buff after skb on the write or rtx queue of sk.  */
tcp_insert_write_queue_after(struct sk_buff * skb,struct sk_buff * buff,struct sock * sk,enum tcp_queue tcp_queue)1527 static void tcp_insert_write_queue_after(struct sk_buff *skb,
1528 					 struct sk_buff *buff,
1529 					 struct sock *sk,
1530 					 enum tcp_queue tcp_queue)
1531 {
1532 	if (tcp_queue == TCP_FRAG_IN_WRITE_QUEUE)
1533 		__skb_queue_after(&sk->sk_write_queue, skb, buff);
1534 	else
1535 		tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
1536 }
1537 
1538 /* Function to create two new TCP segments.  Shrinks the given segment
1539  * to the specified size and appends a new segment with the rest of the
1540  * packet to the list.  This won't be called frequently, I hope.
1541  * Remember, these are still headerless SKBs at this point.
1542  */
tcp_fragment(struct sock * sk,enum tcp_queue tcp_queue,struct sk_buff * skb,u32 len,unsigned int mss_now,gfp_t gfp)1543 int tcp_fragment(struct sock *sk, enum tcp_queue tcp_queue,
1544 		 struct sk_buff *skb, u32 len,
1545 		 unsigned int mss_now, gfp_t gfp)
1546 {
1547 	struct tcp_sock *tp = tcp_sk(sk);
1548 	struct sk_buff *buff;
1549 	int old_factor;
1550 	long limit;
1551 	int nlen;
1552 	u8 flags;
1553 
1554 	if (WARN_ON(len > skb->len))
1555 		return -EINVAL;
1556 
1557 	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1558 
1559 	/* tcp_sendmsg() can overshoot sk_wmem_queued by one full size skb.
1560 	 * We need some allowance to not penalize applications setting small
1561 	 * SO_SNDBUF values.
1562 	 * Also allow first and last skb in retransmit queue to be split.
1563 	 */
1564 	limit = sk->sk_sndbuf + 2 * SKB_TRUESIZE(GSO_LEGACY_MAX_SIZE);
1565 	if (unlikely((sk->sk_wmem_queued >> 1) > limit &&
1566 		     tcp_queue != TCP_FRAG_IN_WRITE_QUEUE &&
1567 		     skb != tcp_rtx_queue_head(sk) &&
1568 		     skb != tcp_rtx_queue_tail(sk))) {
1569 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPWQUEUETOOBIG);
1570 		return -ENOMEM;
1571 	}
1572 
1573 	if (skb_unclone_keeptruesize(skb, gfp))
1574 		return -ENOMEM;
1575 
1576 	/* Get a new skb... force flag on. */
1577 	buff = tcp_stream_alloc_skb(sk, gfp, true);
1578 	if (!buff)
1579 		return -ENOMEM; /* We'll just try again later. */
1580 	skb_copy_decrypted(buff, skb);
1581 	mptcp_skb_ext_copy(buff, skb);
1582 
1583 	sk_wmem_queued_add(sk, buff->truesize);
1584 	sk_mem_charge(sk, buff->truesize);
1585 	nlen = skb->len - len;
1586 	buff->truesize += nlen;
1587 	skb->truesize -= nlen;
1588 
1589 	/* Correct the sequence numbers. */
1590 	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
1591 	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
1592 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
1593 
1594 	/* PSH and FIN should only be set in the second packet. */
1595 	flags = TCP_SKB_CB(skb)->tcp_flags;
1596 	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
1597 	TCP_SKB_CB(buff)->tcp_flags = flags;
1598 	TCP_SKB_CB(buff)->sacked = TCP_SKB_CB(skb)->sacked;
1599 	tcp_skb_fragment_eor(skb, buff);
1600 
1601 	skb_split(skb, buff, len);
1602 
1603 	skb_set_delivery_time(buff, skb->tstamp, true);
1604 	tcp_fragment_tstamp(skb, buff);
1605 
1606 	old_factor = tcp_skb_pcount(skb);
1607 
1608 	/* Fix up tso_factor for both original and new SKB.  */
1609 	tcp_set_skb_tso_segs(skb, mss_now);
1610 	tcp_set_skb_tso_segs(buff, mss_now);
1611 
1612 	/* Update delivered info for the new segment */
1613 	TCP_SKB_CB(buff)->tx = TCP_SKB_CB(skb)->tx;
1614 
1615 	/* If this packet has been sent out already, we must
1616 	 * adjust the various packet counters.
1617 	 */
1618 	if (!before(tp->snd_nxt, TCP_SKB_CB(buff)->end_seq)) {
1619 		int diff = old_factor - tcp_skb_pcount(skb) -
1620 			tcp_skb_pcount(buff);
1621 
1622 		if (diff)
1623 			tcp_adjust_pcount(sk, skb, diff);
1624 	}
1625 
1626 	/* Link BUFF into the send queue. */
1627 	__skb_header_release(buff);
1628 	tcp_insert_write_queue_after(skb, buff, sk, tcp_queue);
1629 	if (tcp_queue == TCP_FRAG_IN_RTX_QUEUE)
1630 		list_add(&buff->tcp_tsorted_anchor, &skb->tcp_tsorted_anchor);
1631 
1632 	return 0;
1633 }
1634 
1635 /* This is similar to __pskb_pull_tail(). The difference is that pulled
1636  * data is not copied, but immediately discarded.
1637  */
__pskb_trim_head(struct sk_buff * skb,int len)1638 static int __pskb_trim_head(struct sk_buff *skb, int len)
1639 {
1640 	struct skb_shared_info *shinfo;
1641 	int i, k, eat;
1642 
1643 	DEBUG_NET_WARN_ON_ONCE(skb_headlen(skb));
1644 	eat = len;
1645 	k = 0;
1646 	shinfo = skb_shinfo(skb);
1647 	for (i = 0; i < shinfo->nr_frags; i++) {
1648 		int size = skb_frag_size(&shinfo->frags[i]);
1649 
1650 		if (size <= eat) {
1651 			skb_frag_unref(skb, i);
1652 			eat -= size;
1653 		} else {
1654 			shinfo->frags[k] = shinfo->frags[i];
1655 			if (eat) {
1656 				skb_frag_off_add(&shinfo->frags[k], eat);
1657 				skb_frag_size_sub(&shinfo->frags[k], eat);
1658 				eat = 0;
1659 			}
1660 			k++;
1661 		}
1662 	}
1663 	shinfo->nr_frags = k;
1664 
1665 	skb->data_len -= len;
1666 	skb->len = skb->data_len;
1667 	return len;
1668 }
1669 
1670 /* Remove acked data from a packet in the transmit queue. */
tcp_trim_head(struct sock * sk,struct sk_buff * skb,u32 len)1671 int tcp_trim_head(struct sock *sk, struct sk_buff *skb, u32 len)
1672 {
1673 	u32 delta_truesize;
1674 
1675 	if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
1676 		return -ENOMEM;
1677 
1678 	delta_truesize = __pskb_trim_head(skb, len);
1679 
1680 	TCP_SKB_CB(skb)->seq += len;
1681 
1682 	skb->truesize	   -= delta_truesize;
1683 	sk_wmem_queued_add(sk, -delta_truesize);
1684 	if (!skb_zcopy_pure(skb))
1685 		sk_mem_uncharge(sk, delta_truesize);
1686 
1687 	/* Any change of skb->len requires recalculation of tso factor. */
1688 	if (tcp_skb_pcount(skb) > 1)
1689 		tcp_set_skb_tso_segs(skb, tcp_skb_mss(skb));
1690 
1691 	return 0;
1692 }
1693 
1694 /* Calculate MSS not accounting any TCP options.  */
__tcp_mtu_to_mss(struct sock * sk,int pmtu)1695 static inline int __tcp_mtu_to_mss(struct sock *sk, int pmtu)
1696 {
1697 	const struct tcp_sock *tp = tcp_sk(sk);
1698 	const struct inet_connection_sock *icsk = inet_csk(sk);
1699 	int mss_now;
1700 
1701 	/* Calculate base mss without TCP options:
1702 	   It is MMS_S - sizeof(tcphdr) of rfc1122
1703 	 */
1704 	mss_now = pmtu - icsk->icsk_af_ops->net_header_len - sizeof(struct tcphdr);
1705 
1706 	/* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1707 	if (icsk->icsk_af_ops->net_frag_header_len) {
1708 		const struct dst_entry *dst = __sk_dst_get(sk);
1709 
1710 		if (dst && dst_allfrag(dst))
1711 			mss_now -= icsk->icsk_af_ops->net_frag_header_len;
1712 	}
1713 
1714 	/* Clamp it (mss_clamp does not include tcp options) */
1715 	if (mss_now > tp->rx_opt.mss_clamp)
1716 		mss_now = tp->rx_opt.mss_clamp;
1717 
1718 	/* Now subtract optional transport overhead */
1719 	mss_now -= icsk->icsk_ext_hdr_len;
1720 
1721 	/* Then reserve room for full set of TCP options and 8 bytes of data */
1722 	mss_now = max(mss_now,
1723 		      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_snd_mss));
1724 	return mss_now;
1725 }
1726 
1727 /* Calculate MSS. Not accounting for SACKs here.  */
tcp_mtu_to_mss(struct sock * sk,int pmtu)1728 int tcp_mtu_to_mss(struct sock *sk, int pmtu)
1729 {
1730 	/* Subtract TCP options size, not including SACKs */
1731 	return __tcp_mtu_to_mss(sk, pmtu) -
1732 	       (tcp_sk(sk)->tcp_header_len - sizeof(struct tcphdr));
1733 }
1734 EXPORT_SYMBOL(tcp_mtu_to_mss);
1735 
1736 /* Inverse of above */
tcp_mss_to_mtu(struct sock * sk,int mss)1737 int tcp_mss_to_mtu(struct sock *sk, int mss)
1738 {
1739 	const struct tcp_sock *tp = tcp_sk(sk);
1740 	const struct inet_connection_sock *icsk = inet_csk(sk);
1741 	int mtu;
1742 
1743 	mtu = mss +
1744 	      tp->tcp_header_len +
1745 	      icsk->icsk_ext_hdr_len +
1746 	      icsk->icsk_af_ops->net_header_len;
1747 
1748 	/* IPv6 adds a frag_hdr in case RTAX_FEATURE_ALLFRAG is set */
1749 	if (icsk->icsk_af_ops->net_frag_header_len) {
1750 		const struct dst_entry *dst = __sk_dst_get(sk);
1751 
1752 		if (dst && dst_allfrag(dst))
1753 			mtu += icsk->icsk_af_ops->net_frag_header_len;
1754 	}
1755 	return mtu;
1756 }
1757 EXPORT_SYMBOL(tcp_mss_to_mtu);
1758 
1759 /* MTU probing init per socket */
tcp_mtup_init(struct sock * sk)1760 void tcp_mtup_init(struct sock *sk)
1761 {
1762 	struct tcp_sock *tp = tcp_sk(sk);
1763 	struct inet_connection_sock *icsk = inet_csk(sk);
1764 	struct net *net = sock_net(sk);
1765 
1766 	icsk->icsk_mtup.enabled = READ_ONCE(net->ipv4.sysctl_tcp_mtu_probing) > 1;
1767 	icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp + sizeof(struct tcphdr) +
1768 			       icsk->icsk_af_ops->net_header_len;
1769 	icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, READ_ONCE(net->ipv4.sysctl_tcp_base_mss));
1770 	icsk->icsk_mtup.probe_size = 0;
1771 	if (icsk->icsk_mtup.enabled)
1772 		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
1773 }
1774 EXPORT_SYMBOL(tcp_mtup_init);
1775 
1776 /* This function synchronize snd mss to current pmtu/exthdr set.
1777 
1778    tp->rx_opt.user_mss is mss set by user by TCP_MAXSEG. It does NOT counts
1779    for TCP options, but includes only bare TCP header.
1780 
1781    tp->rx_opt.mss_clamp is mss negotiated at connection setup.
1782    It is minimum of user_mss and mss received with SYN.
1783    It also does not include TCP options.
1784 
1785    inet_csk(sk)->icsk_pmtu_cookie is last pmtu, seen by this function.
1786 
1787    tp->mss_cache is current effective sending mss, including
1788    all tcp options except for SACKs. It is evaluated,
1789    taking into account current pmtu, but never exceeds
1790    tp->rx_opt.mss_clamp.
1791 
1792    NOTE1. rfc1122 clearly states that advertised MSS
1793    DOES NOT include either tcp or ip options.
1794 
1795    NOTE2. inet_csk(sk)->icsk_pmtu_cookie and tp->mss_cache
1796    are READ ONLY outside this function.		--ANK (980731)
1797  */
tcp_sync_mss(struct sock * sk,u32 pmtu)1798 unsigned int tcp_sync_mss(struct sock *sk, u32 pmtu)
1799 {
1800 	struct tcp_sock *tp = tcp_sk(sk);
1801 	struct inet_connection_sock *icsk = inet_csk(sk);
1802 	int mss_now;
1803 
1804 	if (icsk->icsk_mtup.search_high > pmtu)
1805 		icsk->icsk_mtup.search_high = pmtu;
1806 
1807 	mss_now = tcp_mtu_to_mss(sk, pmtu);
1808 	mss_now = tcp_bound_to_half_wnd(tp, mss_now);
1809 
1810 	/* And store cached results */
1811 	icsk->icsk_pmtu_cookie = pmtu;
1812 	if (icsk->icsk_mtup.enabled)
1813 		mss_now = min(mss_now, tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_low));
1814 	tp->mss_cache = mss_now;
1815 
1816 	return mss_now;
1817 }
1818 EXPORT_SYMBOL(tcp_sync_mss);
1819 
1820 /* Compute the current effective MSS, taking SACKs and IP options,
1821  * and even PMTU discovery events into account.
1822  */
tcp_current_mss(struct sock * sk)1823 unsigned int tcp_current_mss(struct sock *sk)
1824 {
1825 	const struct tcp_sock *tp = tcp_sk(sk);
1826 	const struct dst_entry *dst = __sk_dst_get(sk);
1827 	u32 mss_now;
1828 	unsigned int header_len;
1829 	struct tcp_out_options opts;
1830 	struct tcp_md5sig_key *md5;
1831 
1832 	mss_now = tp->mss_cache;
1833 
1834 	if (dst) {
1835 		u32 mtu = dst_mtu(dst);
1836 		if (mtu != inet_csk(sk)->icsk_pmtu_cookie)
1837 			mss_now = tcp_sync_mss(sk, mtu);
1838 	}
1839 
1840 	header_len = tcp_established_options(sk, NULL, &opts, &md5) +
1841 		     sizeof(struct tcphdr);
1842 	/* The mss_cache is sized based on tp->tcp_header_len, which assumes
1843 	 * some common options. If this is an odd packet (because we have SACK
1844 	 * blocks etc) then our calculated header_len will be different, and
1845 	 * we have to adjust mss_now correspondingly */
1846 	if (header_len != tp->tcp_header_len) {
1847 		int delta = (int) header_len - tp->tcp_header_len;
1848 		mss_now -= delta;
1849 	}
1850 
1851 	return mss_now;
1852 }
1853 
1854 /* RFC2861, slow part. Adjust cwnd, after it was not full during one rto.
1855  * As additional protections, we do not touch cwnd in retransmission phases,
1856  * and if application hit its sndbuf limit recently.
1857  */
tcp_cwnd_application_limited(struct sock * sk)1858 static void tcp_cwnd_application_limited(struct sock *sk)
1859 {
1860 	struct tcp_sock *tp = tcp_sk(sk);
1861 
1862 	if (inet_csk(sk)->icsk_ca_state == TCP_CA_Open &&
1863 	    sk->sk_socket && !test_bit(SOCK_NOSPACE, &sk->sk_socket->flags)) {
1864 		/* Limited by application or receiver window. */
1865 		u32 init_win = tcp_init_cwnd(tp, __sk_dst_get(sk));
1866 		u32 win_used = max(tp->snd_cwnd_used, init_win);
1867 		if (win_used < tcp_snd_cwnd(tp)) {
1868 			tp->snd_ssthresh = tcp_current_ssthresh(sk);
1869 			tcp_snd_cwnd_set(tp, (tcp_snd_cwnd(tp) + win_used) >> 1);
1870 		}
1871 		tp->snd_cwnd_used = 0;
1872 	}
1873 	tp->snd_cwnd_stamp = tcp_jiffies32;
1874 }
1875 
tcp_cwnd_validate(struct sock * sk,bool is_cwnd_limited)1876 static void tcp_cwnd_validate(struct sock *sk, bool is_cwnd_limited)
1877 {
1878 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1879 	struct tcp_sock *tp = tcp_sk(sk);
1880 
1881 	/* Track the strongest available signal of the degree to which the cwnd
1882 	 * is fully utilized. If cwnd-limited then remember that fact for the
1883 	 * current window. If not cwnd-limited then track the maximum number of
1884 	 * outstanding packets in the current window. (If cwnd-limited then we
1885 	 * chose to not update tp->max_packets_out to avoid an extra else
1886 	 * clause with no functional impact.)
1887 	 */
1888 	if (!before(tp->snd_una, tp->cwnd_usage_seq) ||
1889 	    is_cwnd_limited ||
1890 	    (!tp->is_cwnd_limited &&
1891 	     tp->packets_out > tp->max_packets_out)) {
1892 		tp->is_cwnd_limited = is_cwnd_limited;
1893 		tp->max_packets_out = tp->packets_out;
1894 		tp->cwnd_usage_seq = tp->snd_nxt;
1895 	}
1896 
1897 	if (tcp_is_cwnd_limited(sk)) {
1898 		/* Network is feed fully. */
1899 		tp->snd_cwnd_used = 0;
1900 		tp->snd_cwnd_stamp = tcp_jiffies32;
1901 	} else {
1902 		/* Network starves. */
1903 		if (tp->packets_out > tp->snd_cwnd_used)
1904 			tp->snd_cwnd_used = tp->packets_out;
1905 
1906 		if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_slow_start_after_idle) &&
1907 		    (s32)(tcp_jiffies32 - tp->snd_cwnd_stamp) >= inet_csk(sk)->icsk_rto &&
1908 		    !ca_ops->cong_control)
1909 			tcp_cwnd_application_limited(sk);
1910 
1911 		/* The following conditions together indicate the starvation
1912 		 * is caused by insufficient sender buffer:
1913 		 * 1) just sent some data (see tcp_write_xmit)
1914 		 * 2) not cwnd limited (this else condition)
1915 		 * 3) no more data to send (tcp_write_queue_empty())
1916 		 * 4) application is hitting buffer limit (SOCK_NOSPACE)
1917 		 */
1918 		if (tcp_write_queue_empty(sk) && sk->sk_socket &&
1919 		    test_bit(SOCK_NOSPACE, &sk->sk_socket->flags) &&
1920 		    (1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT))
1921 			tcp_chrono_start(sk, TCP_CHRONO_SNDBUF_LIMITED);
1922 	}
1923 }
1924 
1925 /* Minshall's variant of the Nagle send check. */
tcp_minshall_check(const struct tcp_sock * tp)1926 static bool tcp_minshall_check(const struct tcp_sock *tp)
1927 {
1928 	return after(tp->snd_sml, tp->snd_una) &&
1929 		!after(tp->snd_sml, tp->snd_nxt);
1930 }
1931 
1932 /* Update snd_sml if this skb is under mss
1933  * Note that a TSO packet might end with a sub-mss segment
1934  * The test is really :
1935  * if ((skb->len % mss) != 0)
1936  *        tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1937  * But we can avoid doing the divide again given we already have
1938  *  skb_pcount = skb->len / mss_now
1939  */
tcp_minshall_update(struct tcp_sock * tp,unsigned int mss_now,const struct sk_buff * skb)1940 static void tcp_minshall_update(struct tcp_sock *tp, unsigned int mss_now,
1941 				const struct sk_buff *skb)
1942 {
1943 	if (skb->len < tcp_skb_pcount(skb) * mss_now)
1944 		tp->snd_sml = TCP_SKB_CB(skb)->end_seq;
1945 }
1946 
1947 /* Return false, if packet can be sent now without violation Nagle's rules:
1948  * 1. It is full sized. (provided by caller in %partial bool)
1949  * 2. Or it contains FIN. (already checked by caller)
1950  * 3. Or TCP_CORK is not set, and TCP_NODELAY is set.
1951  * 4. Or TCP_CORK is not set, and all sent packets are ACKed.
1952  *    With Minshall's modification: all sent small packets are ACKed.
1953  */
tcp_nagle_check(bool partial,const struct tcp_sock * tp,int nonagle)1954 static bool tcp_nagle_check(bool partial, const struct tcp_sock *tp,
1955 			    int nonagle)
1956 {
1957 	return partial &&
1958 		((nonagle & TCP_NAGLE_CORK) ||
1959 		 (!nonagle && tp->packets_out && tcp_minshall_check(tp)));
1960 }
1961 
1962 /* Return how many segs we'd like on a TSO packet,
1963  * depending on current pacing rate, and how close the peer is.
1964  *
1965  * Rationale is:
1966  * - For close peers, we rather send bigger packets to reduce
1967  *   cpu costs, because occasional losses will be repaired fast.
1968  * - For long distance/rtt flows, we would like to get ACK clocking
1969  *   with 1 ACK per ms.
1970  *
1971  * Use min_rtt to help adapt TSO burst size, with smaller min_rtt resulting
1972  * in bigger TSO bursts. We we cut the RTT-based allowance in half
1973  * for every 2^9 usec (aka 512 us) of RTT, so that the RTT-based allowance
1974  * is below 1500 bytes after 6 * ~500 usec = 3ms.
1975  */
tcp_tso_autosize(const struct sock * sk,unsigned int mss_now,int min_tso_segs)1976 static u32 tcp_tso_autosize(const struct sock *sk, unsigned int mss_now,
1977 			    int min_tso_segs)
1978 {
1979 	unsigned long bytes;
1980 	u32 r;
1981 
1982 	bytes = sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift);
1983 
1984 	r = tcp_min_rtt(tcp_sk(sk)) >> READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_rtt_log);
1985 	if (r < BITS_PER_TYPE(sk->sk_gso_max_size))
1986 		bytes += sk->sk_gso_max_size >> r;
1987 
1988 	bytes = min_t(unsigned long, bytes, sk->sk_gso_max_size);
1989 
1990 	return max_t(u32, bytes / mss_now, min_tso_segs);
1991 }
1992 
1993 /* Return the number of segments we want in the skb we are transmitting.
1994  * See if congestion control module wants to decide; otherwise, autosize.
1995  */
tcp_tso_segs(struct sock * sk,unsigned int mss_now)1996 static u32 tcp_tso_segs(struct sock *sk, unsigned int mss_now)
1997 {
1998 	const struct tcp_congestion_ops *ca_ops = inet_csk(sk)->icsk_ca_ops;
1999 	u32 min_tso, tso_segs;
2000 
2001 	min_tso = ca_ops->min_tso_segs ?
2002 			ca_ops->min_tso_segs(sk) :
2003 			READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_min_tso_segs);
2004 
2005 	tso_segs = tcp_tso_autosize(sk, mss_now, min_tso);
2006 	return min_t(u32, tso_segs, sk->sk_gso_max_segs);
2007 }
2008 
2009 /* Returns the portion of skb which can be sent right away */
tcp_mss_split_point(const struct sock * sk,const struct sk_buff * skb,unsigned int mss_now,unsigned int max_segs,int nonagle)2010 static unsigned int tcp_mss_split_point(const struct sock *sk,
2011 					const struct sk_buff *skb,
2012 					unsigned int mss_now,
2013 					unsigned int max_segs,
2014 					int nonagle)
2015 {
2016 	const struct tcp_sock *tp = tcp_sk(sk);
2017 	u32 partial, needed, window, max_len;
2018 
2019 	window = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2020 	max_len = mss_now * max_segs;
2021 
2022 	if (likely(max_len <= window && skb != tcp_write_queue_tail(sk)))
2023 		return max_len;
2024 
2025 	needed = min(skb->len, window);
2026 
2027 	if (max_len <= needed)
2028 		return max_len;
2029 
2030 	partial = needed % mss_now;
2031 	/* If last segment is not a full MSS, check if Nagle rules allow us
2032 	 * to include this last segment in this skb.
2033 	 * Otherwise, we'll split the skb at last MSS boundary
2034 	 */
2035 	if (tcp_nagle_check(partial != 0, tp, nonagle))
2036 		return needed - partial;
2037 
2038 	return needed;
2039 }
2040 
2041 /* Can at least one segment of SKB be sent right now, according to the
2042  * congestion window rules?  If so, return how many segments are allowed.
2043  */
tcp_cwnd_test(const struct tcp_sock * tp,const struct sk_buff * skb)2044 static inline unsigned int tcp_cwnd_test(const struct tcp_sock *tp,
2045 					 const struct sk_buff *skb)
2046 {
2047 	u32 in_flight, cwnd, halfcwnd;
2048 
2049 	/* Don't be strict about the congestion window for the final FIN.  */
2050 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) &&
2051 	    tcp_skb_pcount(skb) == 1)
2052 		return 1;
2053 
2054 	in_flight = tcp_packets_in_flight(tp);
2055 	cwnd = tcp_snd_cwnd(tp);
2056 	if (in_flight >= cwnd)
2057 		return 0;
2058 
2059 	/* For better scheduling, ensure we have at least
2060 	 * 2 GSO packets in flight.
2061 	 */
2062 	halfcwnd = max(cwnd >> 1, 1U);
2063 	return min(halfcwnd, cwnd - in_flight);
2064 }
2065 
2066 /* Initialize TSO state of a skb.
2067  * This must be invoked the first time we consider transmitting
2068  * SKB onto the wire.
2069  */
tcp_init_tso_segs(struct sk_buff * skb,unsigned int mss_now)2070 static int tcp_init_tso_segs(struct sk_buff *skb, unsigned int mss_now)
2071 {
2072 	int tso_segs = tcp_skb_pcount(skb);
2073 
2074 	if (!tso_segs || (tso_segs > 1 && tcp_skb_mss(skb) != mss_now)) {
2075 		tcp_set_skb_tso_segs(skb, mss_now);
2076 		tso_segs = tcp_skb_pcount(skb);
2077 	}
2078 	return tso_segs;
2079 }
2080 
2081 
2082 /* Return true if the Nagle test allows this packet to be
2083  * sent now.
2084  */
tcp_nagle_test(const struct tcp_sock * tp,const struct sk_buff * skb,unsigned int cur_mss,int nonagle)2085 static inline bool tcp_nagle_test(const struct tcp_sock *tp, const struct sk_buff *skb,
2086 				  unsigned int cur_mss, int nonagle)
2087 {
2088 	/* Nagle rule does not apply to frames, which sit in the middle of the
2089 	 * write_queue (they have no chances to get new data).
2090 	 *
2091 	 * This is implemented in the callers, where they modify the 'nonagle'
2092 	 * argument based upon the location of SKB in the send queue.
2093 	 */
2094 	if (nonagle & TCP_NAGLE_PUSH)
2095 		return true;
2096 
2097 	/* Don't use the nagle rule for urgent data (or for the final FIN). */
2098 	if (tcp_urg_mode(tp) || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN))
2099 		return true;
2100 
2101 	if (!tcp_nagle_check(skb->len < cur_mss, tp, nonagle))
2102 		return true;
2103 
2104 	return false;
2105 }
2106 
2107 /* Does at least the first segment of SKB fit into the send window? */
tcp_snd_wnd_test(const struct tcp_sock * tp,const struct sk_buff * skb,unsigned int cur_mss)2108 static bool tcp_snd_wnd_test(const struct tcp_sock *tp,
2109 			     const struct sk_buff *skb,
2110 			     unsigned int cur_mss)
2111 {
2112 	u32 end_seq = TCP_SKB_CB(skb)->end_seq;
2113 
2114 	if (skb->len > cur_mss)
2115 		end_seq = TCP_SKB_CB(skb)->seq + cur_mss;
2116 
2117 	return !after(end_seq, tcp_wnd_end(tp));
2118 }
2119 
2120 /* Trim TSO SKB to LEN bytes, put the remaining data into a new packet
2121  * which is put after SKB on the list.  It is very much like
2122  * tcp_fragment() except that it may make several kinds of assumptions
2123  * in order to speed up the splitting operation.  In particular, we
2124  * know that all the data is in scatter-gather pages, and that the
2125  * packet has never been sent out before (and thus is not cloned).
2126  */
tso_fragment(struct sock * sk,struct sk_buff * skb,unsigned int len,unsigned int mss_now,gfp_t gfp)2127 static int tso_fragment(struct sock *sk, struct sk_buff *skb, unsigned int len,
2128 			unsigned int mss_now, gfp_t gfp)
2129 {
2130 	int nlen = skb->len - len;
2131 	struct sk_buff *buff;
2132 	u8 flags;
2133 
2134 	/* All of a TSO frame must be composed of paged data.  */
2135 	DEBUG_NET_WARN_ON_ONCE(skb->len != skb->data_len);
2136 
2137 	buff = tcp_stream_alloc_skb(sk, gfp, true);
2138 	if (unlikely(!buff))
2139 		return -ENOMEM;
2140 	skb_copy_decrypted(buff, skb);
2141 	mptcp_skb_ext_copy(buff, skb);
2142 
2143 	sk_wmem_queued_add(sk, buff->truesize);
2144 	sk_mem_charge(sk, buff->truesize);
2145 	buff->truesize += nlen;
2146 	skb->truesize -= nlen;
2147 
2148 	/* Correct the sequence numbers. */
2149 	TCP_SKB_CB(buff)->seq = TCP_SKB_CB(skb)->seq + len;
2150 	TCP_SKB_CB(buff)->end_seq = TCP_SKB_CB(skb)->end_seq;
2151 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(buff)->seq;
2152 
2153 	/* PSH and FIN should only be set in the second packet. */
2154 	flags = TCP_SKB_CB(skb)->tcp_flags;
2155 	TCP_SKB_CB(skb)->tcp_flags = flags & ~(TCPHDR_FIN | TCPHDR_PSH);
2156 	TCP_SKB_CB(buff)->tcp_flags = flags;
2157 
2158 	tcp_skb_fragment_eor(skb, buff);
2159 
2160 	skb_split(skb, buff, len);
2161 	tcp_fragment_tstamp(skb, buff);
2162 
2163 	/* Fix up tso_factor for both original and new SKB.  */
2164 	tcp_set_skb_tso_segs(skb, mss_now);
2165 	tcp_set_skb_tso_segs(buff, mss_now);
2166 
2167 	/* Link BUFF into the send queue. */
2168 	__skb_header_release(buff);
2169 	tcp_insert_write_queue_after(skb, buff, sk, TCP_FRAG_IN_WRITE_QUEUE);
2170 
2171 	return 0;
2172 }
2173 
2174 /* Try to defer sending, if possible, in order to minimize the amount
2175  * of TSO splitting we do.  View it as a kind of TSO Nagle test.
2176  *
2177  * This algorithm is from John Heffner.
2178  */
tcp_tso_should_defer(struct sock * sk,struct sk_buff * skb,bool * is_cwnd_limited,bool * is_rwnd_limited,u32 max_segs)2179 static bool tcp_tso_should_defer(struct sock *sk, struct sk_buff *skb,
2180 				 bool *is_cwnd_limited,
2181 				 bool *is_rwnd_limited,
2182 				 u32 max_segs)
2183 {
2184 	const struct inet_connection_sock *icsk = inet_csk(sk);
2185 	u32 send_win, cong_win, limit, in_flight;
2186 	struct tcp_sock *tp = tcp_sk(sk);
2187 	struct sk_buff *head;
2188 	int win_divisor;
2189 	s64 delta;
2190 
2191 	if (icsk->icsk_ca_state >= TCP_CA_Recovery)
2192 		goto send_now;
2193 
2194 	/* Avoid bursty behavior by allowing defer
2195 	 * only if the last write was recent (1 ms).
2196 	 * Note that tp->tcp_wstamp_ns can be in the future if we have
2197 	 * packets waiting in a qdisc or device for EDT delivery.
2198 	 */
2199 	delta = tp->tcp_clock_cache - tp->tcp_wstamp_ns - NSEC_PER_MSEC;
2200 	if (delta > 0)
2201 		goto send_now;
2202 
2203 	in_flight = tcp_packets_in_flight(tp);
2204 
2205 	BUG_ON(tcp_skb_pcount(skb) <= 1);
2206 	BUG_ON(tcp_snd_cwnd(tp) <= in_flight);
2207 
2208 	send_win = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
2209 
2210 	/* From in_flight test above, we know that cwnd > in_flight.  */
2211 	cong_win = (tcp_snd_cwnd(tp) - in_flight) * tp->mss_cache;
2212 
2213 	limit = min(send_win, cong_win);
2214 
2215 	/* If a full-sized TSO skb can be sent, do it. */
2216 	if (limit >= max_segs * tp->mss_cache)
2217 		goto send_now;
2218 
2219 	/* Middle in queue won't get any more data, full sendable already? */
2220 	if ((skb != tcp_write_queue_tail(sk)) && (limit >= skb->len))
2221 		goto send_now;
2222 
2223 	win_divisor = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tso_win_divisor);
2224 	if (win_divisor) {
2225 		u32 chunk = min(tp->snd_wnd, tcp_snd_cwnd(tp) * tp->mss_cache);
2226 
2227 		/* If at least some fraction of a window is available,
2228 		 * just use it.
2229 		 */
2230 		chunk /= win_divisor;
2231 		if (limit >= chunk)
2232 			goto send_now;
2233 	} else {
2234 		/* Different approach, try not to defer past a single
2235 		 * ACK.  Receiver should ACK every other full sized
2236 		 * frame, so if we have space for more than 3 frames
2237 		 * then send now.
2238 		 */
2239 		if (limit > tcp_max_tso_deferred_mss(tp) * tp->mss_cache)
2240 			goto send_now;
2241 	}
2242 
2243 	/* TODO : use tsorted_sent_queue ? */
2244 	head = tcp_rtx_queue_head(sk);
2245 	if (!head)
2246 		goto send_now;
2247 	delta = tp->tcp_clock_cache - head->tstamp;
2248 	/* If next ACK is likely to come too late (half srtt), do not defer */
2249 	if ((s64)(delta - (u64)NSEC_PER_USEC * (tp->srtt_us >> 4)) < 0)
2250 		goto send_now;
2251 
2252 	/* Ok, it looks like it is advisable to defer.
2253 	 * Three cases are tracked :
2254 	 * 1) We are cwnd-limited
2255 	 * 2) We are rwnd-limited
2256 	 * 3) We are application limited.
2257 	 */
2258 	if (cong_win < send_win) {
2259 		if (cong_win <= skb->len) {
2260 			*is_cwnd_limited = true;
2261 			return true;
2262 		}
2263 	} else {
2264 		if (send_win <= skb->len) {
2265 			*is_rwnd_limited = true;
2266 			return true;
2267 		}
2268 	}
2269 
2270 	/* If this packet won't get more data, do not wait. */
2271 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) ||
2272 	    TCP_SKB_CB(skb)->eor)
2273 		goto send_now;
2274 
2275 	return true;
2276 
2277 send_now:
2278 	return false;
2279 }
2280 
tcp_mtu_check_reprobe(struct sock * sk)2281 static inline void tcp_mtu_check_reprobe(struct sock *sk)
2282 {
2283 	struct inet_connection_sock *icsk = inet_csk(sk);
2284 	struct tcp_sock *tp = tcp_sk(sk);
2285 	struct net *net = sock_net(sk);
2286 	u32 interval;
2287 	s32 delta;
2288 
2289 	interval = READ_ONCE(net->ipv4.sysctl_tcp_probe_interval);
2290 	delta = tcp_jiffies32 - icsk->icsk_mtup.probe_timestamp;
2291 	if (unlikely(delta >= interval * HZ)) {
2292 		int mss = tcp_current_mss(sk);
2293 
2294 		/* Update current search range */
2295 		icsk->icsk_mtup.probe_size = 0;
2296 		icsk->icsk_mtup.search_high = tp->rx_opt.mss_clamp +
2297 			sizeof(struct tcphdr) +
2298 			icsk->icsk_af_ops->net_header_len;
2299 		icsk->icsk_mtup.search_low = tcp_mss_to_mtu(sk, mss);
2300 
2301 		/* Update probe time stamp */
2302 		icsk->icsk_mtup.probe_timestamp = tcp_jiffies32;
2303 	}
2304 }
2305 
tcp_can_coalesce_send_queue_head(struct sock * sk,int len)2306 static bool tcp_can_coalesce_send_queue_head(struct sock *sk, int len)
2307 {
2308 	struct sk_buff *skb, *next;
2309 
2310 	skb = tcp_send_head(sk);
2311 	tcp_for_write_queue_from_safe(skb, next, sk) {
2312 		if (len <= skb->len)
2313 			break;
2314 
2315 		if (unlikely(TCP_SKB_CB(skb)->eor) ||
2316 		    tcp_has_tx_tstamp(skb) ||
2317 		    !skb_pure_zcopy_same(skb, next))
2318 			return false;
2319 
2320 		len -= skb->len;
2321 	}
2322 
2323 	return true;
2324 }
2325 
tcp_clone_payload(struct sock * sk,struct sk_buff * to,int probe_size)2326 static int tcp_clone_payload(struct sock *sk, struct sk_buff *to,
2327 			     int probe_size)
2328 {
2329 	skb_frag_t *lastfrag = NULL, *fragto = skb_shinfo(to)->frags;
2330 	int i, todo, len = 0, nr_frags = 0;
2331 	const struct sk_buff *skb;
2332 
2333 	if (!sk_wmem_schedule(sk, to->truesize + probe_size))
2334 		return -ENOMEM;
2335 
2336 	skb_queue_walk(&sk->sk_write_queue, skb) {
2337 		const skb_frag_t *fragfrom = skb_shinfo(skb)->frags;
2338 
2339 		if (skb_headlen(skb))
2340 			return -EINVAL;
2341 
2342 		for (i = 0; i < skb_shinfo(skb)->nr_frags; i++, fragfrom++) {
2343 			if (len >= probe_size)
2344 				goto commit;
2345 			todo = min_t(int, skb_frag_size(fragfrom),
2346 				     probe_size - len);
2347 			len += todo;
2348 			if (lastfrag &&
2349 			    skb_frag_page(fragfrom) == skb_frag_page(lastfrag) &&
2350 			    skb_frag_off(fragfrom) == skb_frag_off(lastfrag) +
2351 						      skb_frag_size(lastfrag)) {
2352 				skb_frag_size_add(lastfrag, todo);
2353 				continue;
2354 			}
2355 			if (unlikely(nr_frags == MAX_SKB_FRAGS))
2356 				return -E2BIG;
2357 			skb_frag_page_copy(fragto, fragfrom);
2358 			skb_frag_off_copy(fragto, fragfrom);
2359 			skb_frag_size_set(fragto, todo);
2360 			nr_frags++;
2361 			lastfrag = fragto++;
2362 		}
2363 	}
2364 commit:
2365 	WARN_ON_ONCE(len != probe_size);
2366 	for (i = 0; i < nr_frags; i++)
2367 		skb_frag_ref(to, i);
2368 
2369 	skb_shinfo(to)->nr_frags = nr_frags;
2370 	to->truesize += probe_size;
2371 	to->len += probe_size;
2372 	to->data_len += probe_size;
2373 	__skb_header_release(to);
2374 	return 0;
2375 }
2376 
2377 /* Create a new MTU probe if we are ready.
2378  * MTU probe is regularly attempting to increase the path MTU by
2379  * deliberately sending larger packets.  This discovers routing
2380  * changes resulting in larger path MTUs.
2381  *
2382  * Returns 0 if we should wait to probe (no cwnd available),
2383  *         1 if a probe was sent,
2384  *         -1 otherwise
2385  */
tcp_mtu_probe(struct sock * sk)2386 static int tcp_mtu_probe(struct sock *sk)
2387 {
2388 	struct inet_connection_sock *icsk = inet_csk(sk);
2389 	struct tcp_sock *tp = tcp_sk(sk);
2390 	struct sk_buff *skb, *nskb, *next;
2391 	struct net *net = sock_net(sk);
2392 	int probe_size;
2393 	int size_needed;
2394 	int copy, len;
2395 	int mss_now;
2396 	int interval;
2397 
2398 	/* Not currently probing/verifying,
2399 	 * not in recovery,
2400 	 * have enough cwnd, and
2401 	 * not SACKing (the variable headers throw things off)
2402 	 */
2403 	if (likely(!icsk->icsk_mtup.enabled ||
2404 		   icsk->icsk_mtup.probe_size ||
2405 		   inet_csk(sk)->icsk_ca_state != TCP_CA_Open ||
2406 		   tcp_snd_cwnd(tp) < 11 ||
2407 		   tp->rx_opt.num_sacks || tp->rx_opt.dsack))
2408 		return -1;
2409 
2410 	/* Use binary search for probe_size between tcp_mss_base,
2411 	 * and current mss_clamp. if (search_high - search_low)
2412 	 * smaller than a threshold, backoff from probing.
2413 	 */
2414 	mss_now = tcp_current_mss(sk);
2415 	probe_size = tcp_mtu_to_mss(sk, (icsk->icsk_mtup.search_high +
2416 				    icsk->icsk_mtup.search_low) >> 1);
2417 	size_needed = probe_size + (tp->reordering + 1) * tp->mss_cache;
2418 	interval = icsk->icsk_mtup.search_high - icsk->icsk_mtup.search_low;
2419 	/* When misfortune happens, we are reprobing actively,
2420 	 * and then reprobe timer has expired. We stick with current
2421 	 * probing process by not resetting search range to its orignal.
2422 	 */
2423 	if (probe_size > tcp_mtu_to_mss(sk, icsk->icsk_mtup.search_high) ||
2424 	    interval < READ_ONCE(net->ipv4.sysctl_tcp_probe_threshold)) {
2425 		/* Check whether enough time has elaplased for
2426 		 * another round of probing.
2427 		 */
2428 		tcp_mtu_check_reprobe(sk);
2429 		return -1;
2430 	}
2431 
2432 	/* Have enough data in the send queue to probe? */
2433 	if (tp->write_seq - tp->snd_nxt < size_needed)
2434 		return -1;
2435 
2436 	if (tp->snd_wnd < size_needed)
2437 		return -1;
2438 	if (after(tp->snd_nxt + size_needed, tcp_wnd_end(tp)))
2439 		return 0;
2440 
2441 	/* Do we need to wait to drain cwnd? With none in flight, don't stall */
2442 	if (tcp_packets_in_flight(tp) + 2 > tcp_snd_cwnd(tp)) {
2443 		if (!tcp_packets_in_flight(tp))
2444 			return -1;
2445 		else
2446 			return 0;
2447 	}
2448 
2449 	if (!tcp_can_coalesce_send_queue_head(sk, probe_size))
2450 		return -1;
2451 
2452 	/* We're allowed to probe.  Build it now. */
2453 	nskb = tcp_stream_alloc_skb(sk, GFP_ATOMIC, false);
2454 	if (!nskb)
2455 		return -1;
2456 
2457 	/* build the payload, and be prepared to abort if this fails. */
2458 	if (tcp_clone_payload(sk, nskb, probe_size)) {
2459 		tcp_skb_tsorted_anchor_cleanup(nskb);
2460 		consume_skb(nskb);
2461 		return -1;
2462 	}
2463 	sk_wmem_queued_add(sk, nskb->truesize);
2464 	sk_mem_charge(sk, nskb->truesize);
2465 
2466 	skb = tcp_send_head(sk);
2467 	skb_copy_decrypted(nskb, skb);
2468 	mptcp_skb_ext_copy(nskb, skb);
2469 
2470 	TCP_SKB_CB(nskb)->seq = TCP_SKB_CB(skb)->seq;
2471 	TCP_SKB_CB(nskb)->end_seq = TCP_SKB_CB(skb)->seq + probe_size;
2472 	TCP_SKB_CB(nskb)->tcp_flags = TCPHDR_ACK;
2473 
2474 	tcp_insert_write_queue_before(nskb, skb, sk);
2475 	tcp_highest_sack_replace(sk, skb, nskb);
2476 
2477 	len = 0;
2478 	tcp_for_write_queue_from_safe(skb, next, sk) {
2479 		copy = min_t(int, skb->len, probe_size - len);
2480 
2481 		if (skb->len <= copy) {
2482 			/* We've eaten all the data from this skb.
2483 			 * Throw it away. */
2484 			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
2485 			/* If this is the last SKB we copy and eor is set
2486 			 * we need to propagate it to the new skb.
2487 			 */
2488 			TCP_SKB_CB(nskb)->eor = TCP_SKB_CB(skb)->eor;
2489 			tcp_skb_collapse_tstamp(nskb, skb);
2490 			tcp_unlink_write_queue(skb, sk);
2491 			tcp_wmem_free_skb(sk, skb);
2492 		} else {
2493 			TCP_SKB_CB(nskb)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags &
2494 						   ~(TCPHDR_FIN|TCPHDR_PSH);
2495 			__pskb_trim_head(skb, copy);
2496 			tcp_set_skb_tso_segs(skb, mss_now);
2497 			TCP_SKB_CB(skb)->seq += copy;
2498 		}
2499 
2500 		len += copy;
2501 
2502 		if (len >= probe_size)
2503 			break;
2504 	}
2505 	tcp_init_tso_segs(nskb, nskb->len);
2506 
2507 	/* We're ready to send.  If this fails, the probe will
2508 	 * be resegmented into mss-sized pieces by tcp_write_xmit().
2509 	 */
2510 	if (!tcp_transmit_skb(sk, nskb, 1, GFP_ATOMIC)) {
2511 		/* Decrement cwnd here because we are sending
2512 		 * effectively two packets. */
2513 		tcp_snd_cwnd_set(tp, tcp_snd_cwnd(tp) - 1);
2514 		tcp_event_new_data_sent(sk, nskb);
2515 
2516 		icsk->icsk_mtup.probe_size = tcp_mss_to_mtu(sk, nskb->len);
2517 		tp->mtu_probe.probe_seq_start = TCP_SKB_CB(nskb)->seq;
2518 		tp->mtu_probe.probe_seq_end = TCP_SKB_CB(nskb)->end_seq;
2519 
2520 		return 1;
2521 	}
2522 
2523 	return -1;
2524 }
2525 
tcp_pacing_check(struct sock * sk)2526 static bool tcp_pacing_check(struct sock *sk)
2527 {
2528 	struct tcp_sock *tp = tcp_sk(sk);
2529 
2530 	if (!tcp_needs_internal_pacing(sk))
2531 		return false;
2532 
2533 	if (tp->tcp_wstamp_ns <= tp->tcp_clock_cache)
2534 		return false;
2535 
2536 	if (!hrtimer_is_queued(&tp->pacing_timer)) {
2537 		hrtimer_start(&tp->pacing_timer,
2538 			      ns_to_ktime(tp->tcp_wstamp_ns),
2539 			      HRTIMER_MODE_ABS_PINNED_SOFT);
2540 		sock_hold(sk);
2541 	}
2542 	return true;
2543 }
2544 
tcp_rtx_queue_empty_or_single_skb(const struct sock * sk)2545 static bool tcp_rtx_queue_empty_or_single_skb(const struct sock *sk)
2546 {
2547 	const struct rb_node *node = sk->tcp_rtx_queue.rb_node;
2548 
2549 	/* No skb in the rtx queue. */
2550 	if (!node)
2551 		return true;
2552 
2553 	/* Only one skb in rtx queue. */
2554 	return !node->rb_left && !node->rb_right;
2555 }
2556 
2557 /* TCP Small Queues :
2558  * Control number of packets in qdisc/devices to two packets / or ~1 ms.
2559  * (These limits are doubled for retransmits)
2560  * This allows for :
2561  *  - better RTT estimation and ACK scheduling
2562  *  - faster recovery
2563  *  - high rates
2564  * Alas, some drivers / subsystems require a fair amount
2565  * of queued bytes to ensure line rate.
2566  * One example is wifi aggregation (802.11 AMPDU)
2567  */
tcp_small_queue_check(struct sock * sk,const struct sk_buff * skb,unsigned int factor)2568 static bool tcp_small_queue_check(struct sock *sk, const struct sk_buff *skb,
2569 				  unsigned int factor)
2570 {
2571 	unsigned long limit;
2572 
2573 	limit = max_t(unsigned long,
2574 		      2 * skb->truesize,
2575 		      sk->sk_pacing_rate >> READ_ONCE(sk->sk_pacing_shift));
2576 	if (sk->sk_pacing_status == SK_PACING_NONE)
2577 		limit = min_t(unsigned long, limit,
2578 			      READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_limit_output_bytes));
2579 	limit <<= factor;
2580 
2581 	if (static_branch_unlikely(&tcp_tx_delay_enabled) &&
2582 	    tcp_sk(sk)->tcp_tx_delay) {
2583 		u64 extra_bytes = (u64)sk->sk_pacing_rate * tcp_sk(sk)->tcp_tx_delay;
2584 
2585 		/* TSQ is based on skb truesize sum (sk_wmem_alloc), so we
2586 		 * approximate our needs assuming an ~100% skb->truesize overhead.
2587 		 * USEC_PER_SEC is approximated by 2^20.
2588 		 * do_div(extra_bytes, USEC_PER_SEC/2) is replaced by a right shift.
2589 		 */
2590 		extra_bytes >>= (20 - 1);
2591 		limit += extra_bytes;
2592 	}
2593 	if (refcount_read(&sk->sk_wmem_alloc) > limit) {
2594 		/* Always send skb if rtx queue is empty or has one skb.
2595 		 * No need to wait for TX completion to call us back,
2596 		 * after softirq/tasklet schedule.
2597 		 * This helps when TX completions are delayed too much.
2598 		 */
2599 		if (tcp_rtx_queue_empty_or_single_skb(sk))
2600 			return false;
2601 
2602 		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2603 		/* It is possible TX completion already happened
2604 		 * before we set TSQ_THROTTLED, so we must
2605 		 * test again the condition.
2606 		 */
2607 		smp_mb__after_atomic();
2608 		if (refcount_read(&sk->sk_wmem_alloc) > limit)
2609 			return true;
2610 	}
2611 	return false;
2612 }
2613 
tcp_chrono_set(struct tcp_sock * tp,const enum tcp_chrono new)2614 static void tcp_chrono_set(struct tcp_sock *tp, const enum tcp_chrono new)
2615 {
2616 	const u32 now = tcp_jiffies32;
2617 	enum tcp_chrono old = tp->chrono_type;
2618 
2619 	if (old > TCP_CHRONO_UNSPEC)
2620 		tp->chrono_stat[old - 1] += now - tp->chrono_start;
2621 	tp->chrono_start = now;
2622 	tp->chrono_type = new;
2623 }
2624 
tcp_chrono_start(struct sock * sk,const enum tcp_chrono type)2625 void tcp_chrono_start(struct sock *sk, const enum tcp_chrono type)
2626 {
2627 	struct tcp_sock *tp = tcp_sk(sk);
2628 
2629 	/* If there are multiple conditions worthy of tracking in a
2630 	 * chronograph then the highest priority enum takes precedence
2631 	 * over the other conditions. So that if something "more interesting"
2632 	 * starts happening, stop the previous chrono and start a new one.
2633 	 */
2634 	if (type > tp->chrono_type)
2635 		tcp_chrono_set(tp, type);
2636 }
2637 
tcp_chrono_stop(struct sock * sk,const enum tcp_chrono type)2638 void tcp_chrono_stop(struct sock *sk, const enum tcp_chrono type)
2639 {
2640 	struct tcp_sock *tp = tcp_sk(sk);
2641 
2642 
2643 	/* There are multiple conditions worthy of tracking in a
2644 	 * chronograph, so that the highest priority enum takes
2645 	 * precedence over the other conditions (see tcp_chrono_start).
2646 	 * If a condition stops, we only stop chrono tracking if
2647 	 * it's the "most interesting" or current chrono we are
2648 	 * tracking and starts busy chrono if we have pending data.
2649 	 */
2650 	if (tcp_rtx_and_write_queues_empty(sk))
2651 		tcp_chrono_set(tp, TCP_CHRONO_UNSPEC);
2652 	else if (type == tp->chrono_type)
2653 		tcp_chrono_set(tp, TCP_CHRONO_BUSY);
2654 }
2655 
2656 /* This routine writes packets to the network.  It advances the
2657  * send_head.  This happens as incoming acks open up the remote
2658  * window for us.
2659  *
2660  * LARGESEND note: !tcp_urg_mode is overkill, only frames between
2661  * snd_up-64k-mss .. snd_up cannot be large. However, taking into
2662  * account rare use of URG, this is not a big flaw.
2663  *
2664  * Send at most one packet when push_one > 0. Temporarily ignore
2665  * cwnd limit to force at most one packet out when push_one == 2.
2666 
2667  * Returns true, if no segments are in flight and we have queued segments,
2668  * but cannot send anything now because of SWS or another problem.
2669  */
tcp_write_xmit(struct sock * sk,unsigned int mss_now,int nonagle,int push_one,gfp_t gfp)2670 static bool tcp_write_xmit(struct sock *sk, unsigned int mss_now, int nonagle,
2671 			   int push_one, gfp_t gfp)
2672 {
2673 	struct tcp_sock *tp = tcp_sk(sk);
2674 	struct sk_buff *skb;
2675 	unsigned int tso_segs, sent_pkts;
2676 	int cwnd_quota;
2677 	int result;
2678 	bool is_cwnd_limited = false, is_rwnd_limited = false;
2679 	u32 max_segs;
2680 
2681 	sent_pkts = 0;
2682 
2683 	tcp_mstamp_refresh(tp);
2684 	if (!push_one) {
2685 		/* Do MTU probing. */
2686 		result = tcp_mtu_probe(sk);
2687 		if (!result) {
2688 			return false;
2689 		} else if (result > 0) {
2690 			sent_pkts = 1;
2691 		}
2692 	}
2693 
2694 	max_segs = tcp_tso_segs(sk, mss_now);
2695 	while ((skb = tcp_send_head(sk))) {
2696 		unsigned int limit;
2697 
2698 		if (unlikely(tp->repair) && tp->repair_queue == TCP_SEND_QUEUE) {
2699 			/* "skb_mstamp_ns" is used as a start point for the retransmit timer */
2700 			tp->tcp_wstamp_ns = tp->tcp_clock_cache;
2701 			skb_set_delivery_time(skb, tp->tcp_wstamp_ns, true);
2702 			list_move_tail(&skb->tcp_tsorted_anchor, &tp->tsorted_sent_queue);
2703 			tcp_init_tso_segs(skb, mss_now);
2704 			goto repair; /* Skip network transmission */
2705 		}
2706 
2707 		if (tcp_pacing_check(sk))
2708 			break;
2709 
2710 		tso_segs = tcp_init_tso_segs(skb, mss_now);
2711 		BUG_ON(!tso_segs);
2712 
2713 		cwnd_quota = tcp_cwnd_test(tp, skb);
2714 		if (!cwnd_quota) {
2715 			if (push_one == 2)
2716 				/* Force out a loss probe pkt. */
2717 				cwnd_quota = 1;
2718 			else
2719 				break;
2720 		}
2721 
2722 		if (unlikely(!tcp_snd_wnd_test(tp, skb, mss_now))) {
2723 			is_rwnd_limited = true;
2724 			break;
2725 		}
2726 
2727 		if (tso_segs == 1) {
2728 			if (unlikely(!tcp_nagle_test(tp, skb, mss_now,
2729 						     (tcp_skb_is_last(sk, skb) ?
2730 						      nonagle : TCP_NAGLE_PUSH))))
2731 				break;
2732 		} else {
2733 			if (!push_one &&
2734 			    tcp_tso_should_defer(sk, skb, &is_cwnd_limited,
2735 						 &is_rwnd_limited, max_segs))
2736 				break;
2737 		}
2738 
2739 		limit = mss_now;
2740 		if (tso_segs > 1 && !tcp_urg_mode(tp))
2741 			limit = tcp_mss_split_point(sk, skb, mss_now,
2742 						    min_t(unsigned int,
2743 							  cwnd_quota,
2744 							  max_segs),
2745 						    nonagle);
2746 
2747 		if (skb->len > limit &&
2748 		    unlikely(tso_fragment(sk, skb, limit, mss_now, gfp)))
2749 			break;
2750 
2751 		if (tcp_small_queue_check(sk, skb, 0))
2752 			break;
2753 
2754 		/* Argh, we hit an empty skb(), presumably a thread
2755 		 * is sleeping in sendmsg()/sk_stream_wait_memory().
2756 		 * We do not want to send a pure-ack packet and have
2757 		 * a strange looking rtx queue with empty packet(s).
2758 		 */
2759 		if (TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq)
2760 			break;
2761 
2762 		if (unlikely(tcp_transmit_skb(sk, skb, 1, gfp)))
2763 			break;
2764 
2765 repair:
2766 		/* Advance the send_head.  This one is sent out.
2767 		 * This call will increment packets_out.
2768 		 */
2769 		tcp_event_new_data_sent(sk, skb);
2770 
2771 		tcp_minshall_update(tp, mss_now, skb);
2772 		sent_pkts += tcp_skb_pcount(skb);
2773 
2774 		if (push_one)
2775 			break;
2776 	}
2777 
2778 	if (is_rwnd_limited)
2779 		tcp_chrono_start(sk, TCP_CHRONO_RWND_LIMITED);
2780 	else
2781 		tcp_chrono_stop(sk, TCP_CHRONO_RWND_LIMITED);
2782 
2783 	is_cwnd_limited |= (tcp_packets_in_flight(tp) >= tcp_snd_cwnd(tp));
2784 	if (likely(sent_pkts || is_cwnd_limited))
2785 		tcp_cwnd_validate(sk, is_cwnd_limited);
2786 
2787 	if (likely(sent_pkts)) {
2788 		if (tcp_in_cwnd_reduction(sk))
2789 			tp->prr_out += sent_pkts;
2790 
2791 		/* Send one loss probe per tail loss episode. */
2792 		if (push_one != 2)
2793 			tcp_schedule_loss_probe(sk, false);
2794 		return false;
2795 	}
2796 	return !tp->packets_out && !tcp_write_queue_empty(sk);
2797 }
2798 
tcp_schedule_loss_probe(struct sock * sk,bool advancing_rto)2799 bool tcp_schedule_loss_probe(struct sock *sk, bool advancing_rto)
2800 {
2801 	struct inet_connection_sock *icsk = inet_csk(sk);
2802 	struct tcp_sock *tp = tcp_sk(sk);
2803 	u32 timeout, timeout_us, rto_delta_us;
2804 	int early_retrans;
2805 
2806 	/* Don't do any loss probe on a Fast Open connection before 3WHS
2807 	 * finishes.
2808 	 */
2809 	if (rcu_access_pointer(tp->fastopen_rsk))
2810 		return false;
2811 
2812 	early_retrans = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_early_retrans);
2813 	/* Schedule a loss probe in 2*RTT for SACK capable connections
2814 	 * not in loss recovery, that are either limited by cwnd or application.
2815 	 */
2816 	if ((early_retrans != 3 && early_retrans != 4) ||
2817 	    !tp->packets_out || !tcp_is_sack(tp) ||
2818 	    (icsk->icsk_ca_state != TCP_CA_Open &&
2819 	     icsk->icsk_ca_state != TCP_CA_CWR))
2820 		return false;
2821 
2822 	/* Probe timeout is 2*rtt. Add minimum RTO to account
2823 	 * for delayed ack when there's one outstanding packet. If no RTT
2824 	 * sample is available then probe after TCP_TIMEOUT_INIT.
2825 	 */
2826 	if (tp->srtt_us) {
2827 		timeout_us = tp->srtt_us >> 2;
2828 		if (tp->packets_out == 1)
2829 			timeout_us += tcp_rto_min_us(sk);
2830 		else
2831 			timeout_us += TCP_TIMEOUT_MIN_US;
2832 		timeout = usecs_to_jiffies(timeout_us);
2833 	} else {
2834 		timeout = TCP_TIMEOUT_INIT;
2835 	}
2836 
2837 	/* If the RTO formula yields an earlier time, then use that time. */
2838 	rto_delta_us = advancing_rto ?
2839 			jiffies_to_usecs(inet_csk(sk)->icsk_rto) :
2840 			tcp_rto_delta_us(sk);  /* How far in future is RTO? */
2841 	if (rto_delta_us > 0)
2842 		timeout = min_t(u32, timeout, usecs_to_jiffies(rto_delta_us));
2843 
2844 	tcp_reset_xmit_timer(sk, ICSK_TIME_LOSS_PROBE, timeout, TCP_RTO_MAX);
2845 	return true;
2846 }
2847 
2848 /* Thanks to skb fast clones, we can detect if a prior transmit of
2849  * a packet is still in a qdisc or driver queue.
2850  * In this case, there is very little point doing a retransmit !
2851  */
skb_still_in_host_queue(struct sock * sk,const struct sk_buff * skb)2852 static bool skb_still_in_host_queue(struct sock *sk,
2853 				    const struct sk_buff *skb)
2854 {
2855 	if (unlikely(skb_fclone_busy(sk, skb))) {
2856 		set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
2857 		smp_mb__after_atomic();
2858 		if (skb_fclone_busy(sk, skb)) {
2859 			NET_INC_STATS(sock_net(sk),
2860 				      LINUX_MIB_TCPSPURIOUS_RTX_HOSTQUEUES);
2861 			return true;
2862 		}
2863 	}
2864 	return false;
2865 }
2866 
2867 /* When probe timeout (PTO) fires, try send a new segment if possible, else
2868  * retransmit the last segment.
2869  */
tcp_send_loss_probe(struct sock * sk)2870 void tcp_send_loss_probe(struct sock *sk)
2871 {
2872 	struct tcp_sock *tp = tcp_sk(sk);
2873 	struct sk_buff *skb;
2874 	int pcount;
2875 	int mss = tcp_current_mss(sk);
2876 
2877 	/* At most one outstanding TLP */
2878 	if (tp->tlp_high_seq)
2879 		goto rearm_timer;
2880 
2881 	tp->tlp_retrans = 0;
2882 	skb = tcp_send_head(sk);
2883 	if (skb && tcp_snd_wnd_test(tp, skb, mss)) {
2884 		pcount = tp->packets_out;
2885 		tcp_write_xmit(sk, mss, TCP_NAGLE_OFF, 2, GFP_ATOMIC);
2886 		if (tp->packets_out > pcount)
2887 			goto probe_sent;
2888 		goto rearm_timer;
2889 	}
2890 	skb = skb_rb_last(&sk->tcp_rtx_queue);
2891 	if (unlikely(!skb)) {
2892 		WARN_ONCE(tp->packets_out,
2893 			  "invalid inflight: %u state %u cwnd %u mss %d\n",
2894 			  tp->packets_out, sk->sk_state, tcp_snd_cwnd(tp), mss);
2895 		inet_csk(sk)->icsk_pending = 0;
2896 		return;
2897 	}
2898 
2899 	if (skb_still_in_host_queue(sk, skb))
2900 		goto rearm_timer;
2901 
2902 	pcount = tcp_skb_pcount(skb);
2903 	if (WARN_ON(!pcount))
2904 		goto rearm_timer;
2905 
2906 	if ((pcount > 1) && (skb->len > (pcount - 1) * mss)) {
2907 		if (unlikely(tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb,
2908 					  (pcount - 1) * mss, mss,
2909 					  GFP_ATOMIC)))
2910 			goto rearm_timer;
2911 		skb = skb_rb_next(skb);
2912 	}
2913 
2914 	if (WARN_ON(!skb || !tcp_skb_pcount(skb)))
2915 		goto rearm_timer;
2916 
2917 	if (__tcp_retransmit_skb(sk, skb, 1))
2918 		goto rearm_timer;
2919 
2920 	tp->tlp_retrans = 1;
2921 
2922 probe_sent:
2923 	/* Record snd_nxt for loss detection. */
2924 	tp->tlp_high_seq = tp->snd_nxt;
2925 
2926 	NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPLOSSPROBES);
2927 	/* Reset s.t. tcp_rearm_rto will restart timer from now */
2928 	inet_csk(sk)->icsk_pending = 0;
2929 rearm_timer:
2930 	tcp_rearm_rto(sk);
2931 }
2932 
2933 /* Push out any pending frames which were held back due to
2934  * TCP_CORK or attempt at coalescing tiny packets.
2935  * The socket must be locked by the caller.
2936  */
__tcp_push_pending_frames(struct sock * sk,unsigned int cur_mss,int nonagle)2937 void __tcp_push_pending_frames(struct sock *sk, unsigned int cur_mss,
2938 			       int nonagle)
2939 {
2940 	/* If we are closed, the bytes will have to remain here.
2941 	 * In time closedown will finish, we empty the write queue and
2942 	 * all will be happy.
2943 	 */
2944 	if (unlikely(sk->sk_state == TCP_CLOSE))
2945 		return;
2946 
2947 	if (tcp_write_xmit(sk, cur_mss, nonagle, 0,
2948 			   sk_gfp_mask(sk, GFP_ATOMIC)))
2949 		tcp_check_probe_timer(sk);
2950 }
2951 
2952 /* Send _single_ skb sitting at the send head. This function requires
2953  * true push pending frames to setup probe timer etc.
2954  */
tcp_push_one(struct sock * sk,unsigned int mss_now)2955 void tcp_push_one(struct sock *sk, unsigned int mss_now)
2956 {
2957 	struct sk_buff *skb = tcp_send_head(sk);
2958 
2959 	BUG_ON(!skb || skb->len < mss_now);
2960 
2961 	tcp_write_xmit(sk, mss_now, TCP_NAGLE_PUSH, 1, sk->sk_allocation);
2962 }
2963 
2964 /* This function returns the amount that we can raise the
2965  * usable window based on the following constraints
2966  *
2967  * 1. The window can never be shrunk once it is offered (RFC 793)
2968  * 2. We limit memory per socket
2969  *
2970  * RFC 1122:
2971  * "the suggested [SWS] avoidance algorithm for the receiver is to keep
2972  *  RECV.NEXT + RCV.WIN fixed until:
2973  *  RCV.BUFF - RCV.USER - RCV.WINDOW >= min(1/2 RCV.BUFF, MSS)"
2974  *
2975  * i.e. don't raise the right edge of the window until you can raise
2976  * it at least MSS bytes.
2977  *
2978  * Unfortunately, the recommended algorithm breaks header prediction,
2979  * since header prediction assumes th->window stays fixed.
2980  *
2981  * Strictly speaking, keeping th->window fixed violates the receiver
2982  * side SWS prevention criteria. The problem is that under this rule
2983  * a stream of single byte packets will cause the right side of the
2984  * window to always advance by a single byte.
2985  *
2986  * Of course, if the sender implements sender side SWS prevention
2987  * then this will not be a problem.
2988  *
2989  * BSD seems to make the following compromise:
2990  *
2991  *	If the free space is less than the 1/4 of the maximum
2992  *	space available and the free space is less than 1/2 mss,
2993  *	then set the window to 0.
2994  *	[ Actually, bsd uses MSS and 1/4 of maximal _window_ ]
2995  *	Otherwise, just prevent the window from shrinking
2996  *	and from being larger than the largest representable value.
2997  *
2998  * This prevents incremental opening of the window in the regime
2999  * where TCP is limited by the speed of the reader side taking
3000  * data out of the TCP receive queue. It does nothing about
3001  * those cases where the window is constrained on the sender side
3002  * because the pipeline is full.
3003  *
3004  * BSD also seems to "accidentally" limit itself to windows that are a
3005  * multiple of MSS, at least until the free space gets quite small.
3006  * This would appear to be a side effect of the mbuf implementation.
3007  * Combining these two algorithms results in the observed behavior
3008  * of having a fixed window size at almost all times.
3009  *
3010  * Below we obtain similar behavior by forcing the offered window to
3011  * a multiple of the mss when it is feasible to do so.
3012  *
3013  * Note, we don't "adjust" for TIMESTAMP or SACK option bytes.
3014  * Regular options like TIMESTAMP are taken into account.
3015  */
__tcp_select_window(struct sock * sk)3016 u32 __tcp_select_window(struct sock *sk)
3017 {
3018 	struct inet_connection_sock *icsk = inet_csk(sk);
3019 	struct tcp_sock *tp = tcp_sk(sk);
3020 	struct net *net = sock_net(sk);
3021 	/* MSS for the peer's data.  Previous versions used mss_clamp
3022 	 * here.  I don't know if the value based on our guesses
3023 	 * of peer's MSS is better for the performance.  It's more correct
3024 	 * but may be worse for the performance because of rcv_mss
3025 	 * fluctuations.  --SAW  1998/11/1
3026 	 */
3027 	int mss = icsk->icsk_ack.rcv_mss;
3028 	int free_space = tcp_space(sk);
3029 	int allowed_space = tcp_full_space(sk);
3030 	int full_space, window;
3031 
3032 	if (sk_is_mptcp(sk))
3033 		mptcp_space(sk, &free_space, &allowed_space);
3034 
3035 	full_space = min_t(int, tp->window_clamp, allowed_space);
3036 
3037 	if (unlikely(mss > full_space)) {
3038 		mss = full_space;
3039 		if (mss <= 0)
3040 			return 0;
3041 	}
3042 
3043 	/* Only allow window shrink if the sysctl is enabled and we have
3044 	 * a non-zero scaling factor in effect.
3045 	 */
3046 	if (READ_ONCE(net->ipv4.sysctl_tcp_shrink_window) && tp->rx_opt.rcv_wscale)
3047 		goto shrink_window_allowed;
3048 
3049 	/* do not allow window to shrink */
3050 
3051 	if (free_space < (full_space >> 1)) {
3052 		icsk->icsk_ack.quick = 0;
3053 
3054 		if (tcp_under_memory_pressure(sk))
3055 			tcp_adjust_rcv_ssthresh(sk);
3056 
3057 		/* free_space might become our new window, make sure we don't
3058 		 * increase it due to wscale.
3059 		 */
3060 		free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3061 
3062 		/* if free space is less than mss estimate, or is below 1/16th
3063 		 * of the maximum allowed, try to move to zero-window, else
3064 		 * tcp_clamp_window() will grow rcv buf up to tcp_rmem[2], and
3065 		 * new incoming data is dropped due to memory limits.
3066 		 * With large window, mss test triggers way too late in order
3067 		 * to announce zero window in time before rmem limit kicks in.
3068 		 */
3069 		if (free_space < (allowed_space >> 4) || free_space < mss)
3070 			return 0;
3071 	}
3072 
3073 	if (free_space > tp->rcv_ssthresh)
3074 		free_space = tp->rcv_ssthresh;
3075 
3076 	/* Don't do rounding if we are using window scaling, since the
3077 	 * scaled window will not line up with the MSS boundary anyway.
3078 	 */
3079 	if (tp->rx_opt.rcv_wscale) {
3080 		window = free_space;
3081 
3082 		/* Advertise enough space so that it won't get scaled away.
3083 		 * Import case: prevent zero window announcement if
3084 		 * 1<<rcv_wscale > mss.
3085 		 */
3086 		window = ALIGN(window, (1 << tp->rx_opt.rcv_wscale));
3087 	} else {
3088 		window = tp->rcv_wnd;
3089 		/* Get the largest window that is a nice multiple of mss.
3090 		 * Window clamp already applied above.
3091 		 * If our current window offering is within 1 mss of the
3092 		 * free space we just keep it. This prevents the divide
3093 		 * and multiply from happening most of the time.
3094 		 * We also don't do any window rounding when the free space
3095 		 * is too small.
3096 		 */
3097 		if (window <= free_space - mss || window > free_space)
3098 			window = rounddown(free_space, mss);
3099 		else if (mss == full_space &&
3100 			 free_space > window + (full_space >> 1))
3101 			window = free_space;
3102 	}
3103 
3104 	return window;
3105 
3106 shrink_window_allowed:
3107 	/* new window should always be an exact multiple of scaling factor */
3108 	free_space = round_down(free_space, 1 << tp->rx_opt.rcv_wscale);
3109 
3110 	if (free_space < (full_space >> 1)) {
3111 		icsk->icsk_ack.quick = 0;
3112 
3113 		if (tcp_under_memory_pressure(sk))
3114 			tcp_adjust_rcv_ssthresh(sk);
3115 
3116 		/* if free space is too low, return a zero window */
3117 		if (free_space < (allowed_space >> 4) || free_space < mss ||
3118 			free_space < (1 << tp->rx_opt.rcv_wscale))
3119 			return 0;
3120 	}
3121 
3122 	if (free_space > tp->rcv_ssthresh) {
3123 		free_space = tp->rcv_ssthresh;
3124 		/* new window should always be an exact multiple of scaling factor
3125 		 *
3126 		 * For this case, we ALIGN "up" (increase free_space) because
3127 		 * we know free_space is not zero here, it has been reduced from
3128 		 * the memory-based limit, and rcv_ssthresh is not a hard limit
3129 		 * (unlike sk_rcvbuf).
3130 		 */
3131 		free_space = ALIGN(free_space, (1 << tp->rx_opt.rcv_wscale));
3132 	}
3133 
3134 	return free_space;
3135 }
3136 
tcp_skb_collapse_tstamp(struct sk_buff * skb,const struct sk_buff * next_skb)3137 void tcp_skb_collapse_tstamp(struct sk_buff *skb,
3138 			     const struct sk_buff *next_skb)
3139 {
3140 	if (unlikely(tcp_has_tx_tstamp(next_skb))) {
3141 		const struct skb_shared_info *next_shinfo =
3142 			skb_shinfo(next_skb);
3143 		struct skb_shared_info *shinfo = skb_shinfo(skb);
3144 
3145 		shinfo->tx_flags |= next_shinfo->tx_flags & SKBTX_ANY_TSTAMP;
3146 		shinfo->tskey = next_shinfo->tskey;
3147 		TCP_SKB_CB(skb)->txstamp_ack |=
3148 			TCP_SKB_CB(next_skb)->txstamp_ack;
3149 	}
3150 }
3151 
3152 /* Collapses two adjacent SKB's during retransmission. */
tcp_collapse_retrans(struct sock * sk,struct sk_buff * skb)3153 static bool tcp_collapse_retrans(struct sock *sk, struct sk_buff *skb)
3154 {
3155 	struct tcp_sock *tp = tcp_sk(sk);
3156 	struct sk_buff *next_skb = skb_rb_next(skb);
3157 	int next_skb_size;
3158 
3159 	next_skb_size = next_skb->len;
3160 
3161 	BUG_ON(tcp_skb_pcount(skb) != 1 || tcp_skb_pcount(next_skb) != 1);
3162 
3163 	if (next_skb_size && !tcp_skb_shift(skb, next_skb, 1, next_skb_size))
3164 		return false;
3165 
3166 	tcp_highest_sack_replace(sk, next_skb, skb);
3167 
3168 	/* Update sequence range on original skb. */
3169 	TCP_SKB_CB(skb)->end_seq = TCP_SKB_CB(next_skb)->end_seq;
3170 
3171 	/* Merge over control information. This moves PSH/FIN etc. over */
3172 	TCP_SKB_CB(skb)->tcp_flags |= TCP_SKB_CB(next_skb)->tcp_flags;
3173 
3174 	/* All done, get rid of second SKB and account for it so
3175 	 * packet counting does not break.
3176 	 */
3177 	TCP_SKB_CB(skb)->sacked |= TCP_SKB_CB(next_skb)->sacked & TCPCB_EVER_RETRANS;
3178 	TCP_SKB_CB(skb)->eor = TCP_SKB_CB(next_skb)->eor;
3179 
3180 	/* changed transmit queue under us so clear hints */
3181 	tcp_clear_retrans_hints_partial(tp);
3182 	if (next_skb == tp->retransmit_skb_hint)
3183 		tp->retransmit_skb_hint = skb;
3184 
3185 	tcp_adjust_pcount(sk, next_skb, tcp_skb_pcount(next_skb));
3186 
3187 	tcp_skb_collapse_tstamp(skb, next_skb);
3188 
3189 	tcp_rtx_queue_unlink_and_free(next_skb, sk);
3190 	return true;
3191 }
3192 
3193 /* Check if coalescing SKBs is legal. */
tcp_can_collapse(const struct sock * sk,const struct sk_buff * skb)3194 static bool tcp_can_collapse(const struct sock *sk, const struct sk_buff *skb)
3195 {
3196 	if (tcp_skb_pcount(skb) > 1)
3197 		return false;
3198 	if (skb_cloned(skb))
3199 		return false;
3200 	/* Some heuristics for collapsing over SACK'd could be invented */
3201 	if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_ACKED)
3202 		return false;
3203 
3204 	return true;
3205 }
3206 
3207 /* Collapse packets in the retransmit queue to make to create
3208  * less packets on the wire. This is only done on retransmission.
3209  */
tcp_retrans_try_collapse(struct sock * sk,struct sk_buff * to,int space)3210 static void tcp_retrans_try_collapse(struct sock *sk, struct sk_buff *to,
3211 				     int space)
3212 {
3213 	struct tcp_sock *tp = tcp_sk(sk);
3214 	struct sk_buff *skb = to, *tmp;
3215 	bool first = true;
3216 
3217 	if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_retrans_collapse))
3218 		return;
3219 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3220 		return;
3221 
3222 	skb_rbtree_walk_from_safe(skb, tmp) {
3223 		if (!tcp_can_collapse(sk, skb))
3224 			break;
3225 
3226 		if (!tcp_skb_can_collapse(to, skb))
3227 			break;
3228 
3229 		space -= skb->len;
3230 
3231 		if (first) {
3232 			first = false;
3233 			continue;
3234 		}
3235 
3236 		if (space < 0)
3237 			break;
3238 
3239 		if (after(TCP_SKB_CB(skb)->end_seq, tcp_wnd_end(tp)))
3240 			break;
3241 
3242 		if (!tcp_collapse_retrans(sk, to))
3243 			break;
3244 	}
3245 }
3246 
3247 /* This retransmits one SKB.  Policy decisions and retransmit queue
3248  * state updates are done by the caller.  Returns non-zero if an
3249  * error occurred which prevented the send.
3250  */
__tcp_retransmit_skb(struct sock * sk,struct sk_buff * skb,int segs)3251 int __tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3252 {
3253 	struct inet_connection_sock *icsk = inet_csk(sk);
3254 	struct tcp_sock *tp = tcp_sk(sk);
3255 	unsigned int cur_mss;
3256 	int diff, len, err;
3257 	int avail_wnd;
3258 
3259 	/* Inconclusive MTU probe */
3260 	if (icsk->icsk_mtup.probe_size)
3261 		icsk->icsk_mtup.probe_size = 0;
3262 
3263 	if (skb_still_in_host_queue(sk, skb))
3264 		return -EBUSY;
3265 
3266 start:
3267 	if (before(TCP_SKB_CB(skb)->seq, tp->snd_una)) {
3268 		if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3269 			TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_SYN;
3270 			TCP_SKB_CB(skb)->seq++;
3271 			goto start;
3272 		}
3273 		if (unlikely(before(TCP_SKB_CB(skb)->end_seq, tp->snd_una))) {
3274 			WARN_ON_ONCE(1);
3275 			return -EINVAL;
3276 		}
3277 		if (tcp_trim_head(sk, skb, tp->snd_una - TCP_SKB_CB(skb)->seq))
3278 			return -ENOMEM;
3279 	}
3280 
3281 	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3282 		return -EHOSTUNREACH; /* Routing failure or similar. */
3283 
3284 	cur_mss = tcp_current_mss(sk);
3285 	avail_wnd = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
3286 
3287 	/* If receiver has shrunk his window, and skb is out of
3288 	 * new window, do not retransmit it. The exception is the
3289 	 * case, when window is shrunk to zero. In this case
3290 	 * our retransmit of one segment serves as a zero window probe.
3291 	 */
3292 	if (avail_wnd <= 0) {
3293 		if (TCP_SKB_CB(skb)->seq != tp->snd_una)
3294 			return -EAGAIN;
3295 		avail_wnd = cur_mss;
3296 	}
3297 
3298 	len = cur_mss * segs;
3299 	if (len > avail_wnd) {
3300 		len = rounddown(avail_wnd, cur_mss);
3301 		if (!len)
3302 			len = avail_wnd;
3303 	}
3304 	if (skb->len > len) {
3305 		if (tcp_fragment(sk, TCP_FRAG_IN_RTX_QUEUE, skb, len,
3306 				 cur_mss, GFP_ATOMIC))
3307 			return -ENOMEM; /* We'll try again later. */
3308 	} else {
3309 		if (skb_unclone_keeptruesize(skb, GFP_ATOMIC))
3310 			return -ENOMEM;
3311 
3312 		diff = tcp_skb_pcount(skb);
3313 		tcp_set_skb_tso_segs(skb, cur_mss);
3314 		diff -= tcp_skb_pcount(skb);
3315 		if (diff)
3316 			tcp_adjust_pcount(sk, skb, diff);
3317 		avail_wnd = min_t(int, avail_wnd, cur_mss);
3318 		if (skb->len < avail_wnd)
3319 			tcp_retrans_try_collapse(sk, skb, avail_wnd);
3320 	}
3321 
3322 	/* RFC3168, section 6.1.1.1. ECN fallback */
3323 	if ((TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN_ECN) == TCPHDR_SYN_ECN)
3324 		tcp_ecn_clear_syn(sk, skb);
3325 
3326 	/* Update global and local TCP statistics. */
3327 	segs = tcp_skb_pcount(skb);
3328 	TCP_ADD_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS, segs);
3329 	if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
3330 		__NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
3331 	tp->total_retrans += segs;
3332 	tp->bytes_retrans += skb->len;
3333 
3334 	/* make sure skb->data is aligned on arches that require it
3335 	 * and check if ack-trimming & collapsing extended the headroom
3336 	 * beyond what csum_start can cover.
3337 	 */
3338 	if (unlikely((NET_IP_ALIGN && ((unsigned long)skb->data & 3)) ||
3339 		     skb_headroom(skb) >= 0xFFFF)) {
3340 		struct sk_buff *nskb;
3341 
3342 		tcp_skb_tsorted_save(skb) {
3343 			nskb = __pskb_copy(skb, MAX_TCP_HEADER, GFP_ATOMIC);
3344 			if (nskb) {
3345 				nskb->dev = NULL;
3346 				err = tcp_transmit_skb(sk, nskb, 0, GFP_ATOMIC);
3347 			} else {
3348 				err = -ENOBUFS;
3349 			}
3350 		} tcp_skb_tsorted_restore(skb);
3351 
3352 		if (!err) {
3353 			tcp_update_skb_after_send(sk, skb, tp->tcp_wstamp_ns);
3354 			tcp_rate_skb_sent(sk, skb);
3355 		}
3356 	} else {
3357 		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3358 	}
3359 
3360 	/* To avoid taking spuriously low RTT samples based on a timestamp
3361 	 * for a transmit that never happened, always mark EVER_RETRANS
3362 	 */
3363 	TCP_SKB_CB(skb)->sacked |= TCPCB_EVER_RETRANS;
3364 
3365 	if (BPF_SOCK_OPS_TEST_FLAG(tp, BPF_SOCK_OPS_RETRANS_CB_FLAG))
3366 		tcp_call_bpf_3arg(sk, BPF_SOCK_OPS_RETRANS_CB,
3367 				  TCP_SKB_CB(skb)->seq, segs, err);
3368 
3369 	if (likely(!err)) {
3370 		trace_tcp_retransmit_skb(sk, skb);
3371 	} else if (err != -EBUSY) {
3372 		NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPRETRANSFAIL, segs);
3373 	}
3374 	return err;
3375 }
3376 
tcp_retransmit_skb(struct sock * sk,struct sk_buff * skb,int segs)3377 int tcp_retransmit_skb(struct sock *sk, struct sk_buff *skb, int segs)
3378 {
3379 	struct tcp_sock *tp = tcp_sk(sk);
3380 	int err = __tcp_retransmit_skb(sk, skb, segs);
3381 
3382 	if (err == 0) {
3383 #if FASTRETRANS_DEBUG > 0
3384 		if (TCP_SKB_CB(skb)->sacked & TCPCB_SACKED_RETRANS) {
3385 			net_dbg_ratelimited("retrans_out leaked\n");
3386 		}
3387 #endif
3388 		TCP_SKB_CB(skb)->sacked |= TCPCB_RETRANS;
3389 		tp->retrans_out += tcp_skb_pcount(skb);
3390 	}
3391 
3392 	/* Save stamp of the first (attempted) retransmit. */
3393 	if (!tp->retrans_stamp)
3394 		tp->retrans_stamp = tcp_skb_timestamp(skb);
3395 
3396 	if (tp->undo_retrans < 0)
3397 		tp->undo_retrans = 0;
3398 	tp->undo_retrans += tcp_skb_pcount(skb);
3399 	return err;
3400 }
3401 
3402 /* This gets called after a retransmit timeout, and the initially
3403  * retransmitted data is acknowledged.  It tries to continue
3404  * resending the rest of the retransmit queue, until either
3405  * we've sent it all or the congestion window limit is reached.
3406  */
tcp_xmit_retransmit_queue(struct sock * sk)3407 void tcp_xmit_retransmit_queue(struct sock *sk)
3408 {
3409 	const struct inet_connection_sock *icsk = inet_csk(sk);
3410 	struct sk_buff *skb, *rtx_head, *hole = NULL;
3411 	struct tcp_sock *tp = tcp_sk(sk);
3412 	bool rearm_timer = false;
3413 	u32 max_segs;
3414 	int mib_idx;
3415 
3416 	if (!tp->packets_out)
3417 		return;
3418 
3419 	rtx_head = tcp_rtx_queue_head(sk);
3420 	skb = tp->retransmit_skb_hint ?: rtx_head;
3421 	max_segs = tcp_tso_segs(sk, tcp_current_mss(sk));
3422 	skb_rbtree_walk_from(skb) {
3423 		__u8 sacked;
3424 		int segs;
3425 
3426 		if (tcp_pacing_check(sk))
3427 			break;
3428 
3429 		/* we could do better than to assign each time */
3430 		if (!hole)
3431 			tp->retransmit_skb_hint = skb;
3432 
3433 		segs = tcp_snd_cwnd(tp) - tcp_packets_in_flight(tp);
3434 		if (segs <= 0)
3435 			break;
3436 		sacked = TCP_SKB_CB(skb)->sacked;
3437 		/* In case tcp_shift_skb_data() have aggregated large skbs,
3438 		 * we need to make sure not sending too bigs TSO packets
3439 		 */
3440 		segs = min_t(int, segs, max_segs);
3441 
3442 		if (tp->retrans_out >= tp->lost_out) {
3443 			break;
3444 		} else if (!(sacked & TCPCB_LOST)) {
3445 			if (!hole && !(sacked & (TCPCB_SACKED_RETRANS|TCPCB_SACKED_ACKED)))
3446 				hole = skb;
3447 			continue;
3448 
3449 		} else {
3450 			if (icsk->icsk_ca_state != TCP_CA_Loss)
3451 				mib_idx = LINUX_MIB_TCPFASTRETRANS;
3452 			else
3453 				mib_idx = LINUX_MIB_TCPSLOWSTARTRETRANS;
3454 		}
3455 
3456 		if (sacked & (TCPCB_SACKED_ACKED|TCPCB_SACKED_RETRANS))
3457 			continue;
3458 
3459 		if (tcp_small_queue_check(sk, skb, 1))
3460 			break;
3461 
3462 		if (tcp_retransmit_skb(sk, skb, segs))
3463 			break;
3464 
3465 		NET_ADD_STATS(sock_net(sk), mib_idx, tcp_skb_pcount(skb));
3466 
3467 		if (tcp_in_cwnd_reduction(sk))
3468 			tp->prr_out += tcp_skb_pcount(skb);
3469 
3470 		if (skb == rtx_head &&
3471 		    icsk->icsk_pending != ICSK_TIME_REO_TIMEOUT)
3472 			rearm_timer = true;
3473 
3474 	}
3475 	if (rearm_timer)
3476 		tcp_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
3477 				     inet_csk(sk)->icsk_rto,
3478 				     TCP_RTO_MAX);
3479 }
3480 
3481 /* We allow to exceed memory limits for FIN packets to expedite
3482  * connection tear down and (memory) recovery.
3483  * Otherwise tcp_send_fin() could be tempted to either delay FIN
3484  * or even be forced to close flow without any FIN.
3485  * In general, we want to allow one skb per socket to avoid hangs
3486  * with edge trigger epoll()
3487  */
sk_forced_mem_schedule(struct sock * sk,int size)3488 void sk_forced_mem_schedule(struct sock *sk, int size)
3489 {
3490 	int delta, amt;
3491 
3492 	delta = size - sk->sk_forward_alloc;
3493 	if (delta <= 0)
3494 		return;
3495 	amt = sk_mem_pages(delta);
3496 	sk_forward_alloc_add(sk, amt << PAGE_SHIFT);
3497 	sk_memory_allocated_add(sk, amt);
3498 
3499 	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3500 		mem_cgroup_charge_skmem(sk->sk_memcg, amt,
3501 					gfp_memcg_charge() | __GFP_NOFAIL);
3502 }
3503 
3504 /* Send a FIN. The caller locks the socket for us.
3505  * We should try to send a FIN packet really hard, but eventually give up.
3506  */
tcp_send_fin(struct sock * sk)3507 void tcp_send_fin(struct sock *sk)
3508 {
3509 	struct sk_buff *skb, *tskb, *tail = tcp_write_queue_tail(sk);
3510 	struct tcp_sock *tp = tcp_sk(sk);
3511 
3512 	/* Optimization, tack on the FIN if we have one skb in write queue and
3513 	 * this skb was not yet sent, or we are under memory pressure.
3514 	 * Note: in the latter case, FIN packet will be sent after a timeout,
3515 	 * as TCP stack thinks it has already been transmitted.
3516 	 */
3517 	tskb = tail;
3518 	if (!tskb && tcp_under_memory_pressure(sk))
3519 		tskb = skb_rb_last(&sk->tcp_rtx_queue);
3520 
3521 	if (tskb) {
3522 		TCP_SKB_CB(tskb)->tcp_flags |= TCPHDR_FIN;
3523 		TCP_SKB_CB(tskb)->end_seq++;
3524 		tp->write_seq++;
3525 		if (!tail) {
3526 			/* This means tskb was already sent.
3527 			 * Pretend we included the FIN on previous transmit.
3528 			 * We need to set tp->snd_nxt to the value it would have
3529 			 * if FIN had been sent. This is because retransmit path
3530 			 * does not change tp->snd_nxt.
3531 			 */
3532 			WRITE_ONCE(tp->snd_nxt, tp->snd_nxt + 1);
3533 			return;
3534 		}
3535 	} else {
3536 		skb = alloc_skb_fclone(MAX_TCP_HEADER, sk->sk_allocation);
3537 		if (unlikely(!skb))
3538 			return;
3539 
3540 		INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
3541 		skb_reserve(skb, MAX_TCP_HEADER);
3542 		sk_forced_mem_schedule(sk, skb->truesize);
3543 		/* FIN eats a sequence byte, write_seq advanced by tcp_queue_skb(). */
3544 		tcp_init_nondata_skb(skb, tp->write_seq,
3545 				     TCPHDR_ACK | TCPHDR_FIN);
3546 		tcp_queue_skb(sk, skb);
3547 	}
3548 	__tcp_push_pending_frames(sk, tcp_current_mss(sk), TCP_NAGLE_OFF);
3549 }
3550 
3551 /* We get here when a process closes a file descriptor (either due to
3552  * an explicit close() or as a byproduct of exit()'ing) and there
3553  * was unread data in the receive queue.  This behavior is recommended
3554  * by RFC 2525, section 2.17.  -DaveM
3555  */
tcp_send_active_reset(struct sock * sk,gfp_t priority)3556 void tcp_send_active_reset(struct sock *sk, gfp_t priority)
3557 {
3558 	struct sk_buff *skb;
3559 
3560 	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTRSTS);
3561 
3562 	/* NOTE: No TCP options attached and we never retransmit this. */
3563 	skb = alloc_skb(MAX_TCP_HEADER, priority);
3564 	if (!skb) {
3565 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3566 		return;
3567 	}
3568 
3569 	/* Reserve space for headers and prepare control bits. */
3570 	skb_reserve(skb, MAX_TCP_HEADER);
3571 	tcp_init_nondata_skb(skb, tcp_acceptable_seq(sk),
3572 			     TCPHDR_ACK | TCPHDR_RST);
3573 	tcp_mstamp_refresh(tcp_sk(sk));
3574 	/* Send it off. */
3575 	if (tcp_transmit_skb(sk, skb, 0, priority))
3576 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTFAILED);
3577 
3578 	/* skb of trace_tcp_send_reset() keeps the skb that caused RST,
3579 	 * skb here is different to the troublesome skb, so use NULL
3580 	 */
3581 	trace_tcp_send_reset(sk, NULL);
3582 }
3583 
3584 /* Send a crossed SYN-ACK during socket establishment.
3585  * WARNING: This routine must only be called when we have already sent
3586  * a SYN packet that crossed the incoming SYN that caused this routine
3587  * to get called. If this assumption fails then the initial rcv_wnd
3588  * and rcv_wscale values will not be correct.
3589  */
tcp_send_synack(struct sock * sk)3590 int tcp_send_synack(struct sock *sk)
3591 {
3592 	struct sk_buff *skb;
3593 
3594 	skb = tcp_rtx_queue_head(sk);
3595 	if (!skb || !(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
3596 		pr_err("%s: wrong queue state\n", __func__);
3597 		return -EFAULT;
3598 	}
3599 	if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)) {
3600 		if (skb_cloned(skb)) {
3601 			struct sk_buff *nskb;
3602 
3603 			tcp_skb_tsorted_save(skb) {
3604 				nskb = skb_copy(skb, GFP_ATOMIC);
3605 			} tcp_skb_tsorted_restore(skb);
3606 			if (!nskb)
3607 				return -ENOMEM;
3608 			INIT_LIST_HEAD(&nskb->tcp_tsorted_anchor);
3609 			tcp_highest_sack_replace(sk, skb, nskb);
3610 			tcp_rtx_queue_unlink_and_free(skb, sk);
3611 			__skb_header_release(nskb);
3612 			tcp_rbtree_insert(&sk->tcp_rtx_queue, nskb);
3613 			sk_wmem_queued_add(sk, nskb->truesize);
3614 			sk_mem_charge(sk, nskb->truesize);
3615 			skb = nskb;
3616 		}
3617 
3618 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_ACK;
3619 		tcp_ecn_send_synack(sk, skb);
3620 	}
3621 	return tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
3622 }
3623 
3624 /**
3625  * tcp_make_synack - Allocate one skb and build a SYNACK packet.
3626  * @sk: listener socket
3627  * @dst: dst entry attached to the SYNACK. It is consumed and caller
3628  *       should not use it again.
3629  * @req: request_sock pointer
3630  * @foc: cookie for tcp fast open
3631  * @synack_type: Type of synack to prepare
3632  * @syn_skb: SYN packet just received.  It could be NULL for rtx case.
3633  */
tcp_make_synack(const struct sock * sk,struct dst_entry * dst,struct request_sock * req,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)3634 struct sk_buff *tcp_make_synack(const struct sock *sk, struct dst_entry *dst,
3635 				struct request_sock *req,
3636 				struct tcp_fastopen_cookie *foc,
3637 				enum tcp_synack_type synack_type,
3638 				struct sk_buff *syn_skb)
3639 {
3640 	struct inet_request_sock *ireq = inet_rsk(req);
3641 	const struct tcp_sock *tp = tcp_sk(sk);
3642 	struct tcp_md5sig_key *md5 = NULL;
3643 	struct tcp_out_options opts;
3644 	struct sk_buff *skb;
3645 	int tcp_header_size;
3646 	struct tcphdr *th;
3647 	int mss;
3648 	u64 now;
3649 
3650 	skb = alloc_skb(MAX_TCP_HEADER, GFP_ATOMIC);
3651 	if (unlikely(!skb)) {
3652 		dst_release(dst);
3653 		return NULL;
3654 	}
3655 	/* Reserve space for headers. */
3656 	skb_reserve(skb, MAX_TCP_HEADER);
3657 
3658 	switch (synack_type) {
3659 	case TCP_SYNACK_NORMAL:
3660 		skb_set_owner_w(skb, req_to_sk(req));
3661 		break;
3662 	case TCP_SYNACK_COOKIE:
3663 		/* Under synflood, we do not attach skb to a socket,
3664 		 * to avoid false sharing.
3665 		 */
3666 		break;
3667 	case TCP_SYNACK_FASTOPEN:
3668 		/* sk is a const pointer, because we want to express multiple
3669 		 * cpu might call us concurrently.
3670 		 * sk->sk_wmem_alloc in an atomic, we can promote to rw.
3671 		 */
3672 		skb_set_owner_w(skb, (struct sock *)sk);
3673 		break;
3674 	}
3675 	skb_dst_set(skb, dst);
3676 
3677 	mss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3678 
3679 	memset(&opts, 0, sizeof(opts));
3680 	now = tcp_clock_ns();
3681 #ifdef CONFIG_SYN_COOKIES
3682 	if (unlikely(synack_type == TCP_SYNACK_COOKIE && ireq->tstamp_ok))
3683 		skb_set_delivery_time(skb, cookie_init_timestamp(req, now),
3684 				      true);
3685 	else
3686 #endif
3687 	{
3688 		skb_set_delivery_time(skb, now, true);
3689 		if (!tcp_rsk(req)->snt_synack) /* Timestamp first SYNACK */
3690 			tcp_rsk(req)->snt_synack = tcp_skb_timestamp_us(skb);
3691 	}
3692 
3693 #ifdef CONFIG_TCP_MD5SIG
3694 	rcu_read_lock();
3695 	md5 = tcp_rsk(req)->af_specific->req_md5_lookup(sk, req_to_sk(req));
3696 #endif
3697 	skb_set_hash(skb, READ_ONCE(tcp_rsk(req)->txhash), PKT_HASH_TYPE_L4);
3698 	/* bpf program will be interested in the tcp_flags */
3699 	TCP_SKB_CB(skb)->tcp_flags = TCPHDR_SYN | TCPHDR_ACK;
3700 	tcp_header_size = tcp_synack_options(sk, req, mss, skb, &opts, md5,
3701 					     foc, synack_type,
3702 					     syn_skb) + sizeof(*th);
3703 
3704 	skb_push(skb, tcp_header_size);
3705 	skb_reset_transport_header(skb);
3706 
3707 	th = (struct tcphdr *)skb->data;
3708 	memset(th, 0, sizeof(struct tcphdr));
3709 	th->syn = 1;
3710 	th->ack = 1;
3711 	tcp_ecn_make_synack(req, th);
3712 	th->source = htons(ireq->ir_num);
3713 	th->dest = ireq->ir_rmt_port;
3714 	skb->mark = ireq->ir_mark;
3715 	skb->ip_summed = CHECKSUM_PARTIAL;
3716 	th->seq = htonl(tcp_rsk(req)->snt_isn);
3717 	/* XXX data is queued and acked as is. No buffer/window check */
3718 	th->ack_seq = htonl(tcp_rsk(req)->rcv_nxt);
3719 
3720 	/* RFC1323: The window in SYN & SYN/ACK segments is never scaled. */
3721 	th->window = htons(min(req->rsk_rcv_wnd, 65535U));
3722 	tcp_options_write(th, NULL, &opts);
3723 	th->doff = (tcp_header_size >> 2);
3724 	TCP_INC_STATS(sock_net(sk), TCP_MIB_OUTSEGS);
3725 
3726 #ifdef CONFIG_TCP_MD5SIG
3727 	/* Okay, we have all we need - do the md5 hash if needed */
3728 	if (md5)
3729 		tcp_rsk(req)->af_specific->calc_md5_hash(opts.hash_location,
3730 					       md5, req_to_sk(req), skb);
3731 	rcu_read_unlock();
3732 #endif
3733 
3734 	bpf_skops_write_hdr_opt((struct sock *)sk, skb, req, syn_skb,
3735 				synack_type, &opts);
3736 
3737 	skb_set_delivery_time(skb, now, true);
3738 	tcp_add_tx_delay(skb, tp);
3739 
3740 	return skb;
3741 }
3742 EXPORT_SYMBOL(tcp_make_synack);
3743 
tcp_ca_dst_init(struct sock * sk,const struct dst_entry * dst)3744 static void tcp_ca_dst_init(struct sock *sk, const struct dst_entry *dst)
3745 {
3746 	struct inet_connection_sock *icsk = inet_csk(sk);
3747 	const struct tcp_congestion_ops *ca;
3748 	u32 ca_key = dst_metric(dst, RTAX_CC_ALGO);
3749 
3750 	if (ca_key == TCP_CA_UNSPEC)
3751 		return;
3752 
3753 	rcu_read_lock();
3754 	ca = tcp_ca_find_key(ca_key);
3755 	if (likely(ca && bpf_try_module_get(ca, ca->owner))) {
3756 		bpf_module_put(icsk->icsk_ca_ops, icsk->icsk_ca_ops->owner);
3757 		icsk->icsk_ca_dst_locked = tcp_ca_dst_locked(dst);
3758 		icsk->icsk_ca_ops = ca;
3759 	}
3760 	rcu_read_unlock();
3761 }
3762 
3763 /* Do all connect socket setups that can be done AF independent. */
tcp_connect_init(struct sock * sk)3764 static void tcp_connect_init(struct sock *sk)
3765 {
3766 	const struct dst_entry *dst = __sk_dst_get(sk);
3767 	struct tcp_sock *tp = tcp_sk(sk);
3768 	__u8 rcv_wscale;
3769 	u32 rcv_wnd;
3770 
3771 	/* We'll fix this up when we get a response from the other end.
3772 	 * See tcp_input.c:tcp_rcv_state_process case TCP_SYN_SENT.
3773 	 */
3774 	tp->tcp_header_len = sizeof(struct tcphdr);
3775 	if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_timestamps))
3776 		tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
3777 
3778 	/* If user gave his TCP_MAXSEG, record it to clamp */
3779 	if (tp->rx_opt.user_mss)
3780 		tp->rx_opt.mss_clamp = tp->rx_opt.user_mss;
3781 	tp->max_window = 0;
3782 	tcp_mtup_init(sk);
3783 	tcp_sync_mss(sk, dst_mtu(dst));
3784 
3785 	tcp_ca_dst_init(sk, dst);
3786 
3787 	if (!tp->window_clamp)
3788 		tp->window_clamp = dst_metric(dst, RTAX_WINDOW);
3789 	tp->advmss = tcp_mss_clamp(tp, dst_metric_advmss(dst));
3790 
3791 	tcp_initialize_rcv_mss(sk);
3792 
3793 	/* limit the window selection if the user enforce a smaller rx buffer */
3794 	if (sk->sk_userlocks & SOCK_RCVBUF_LOCK &&
3795 	    (tp->window_clamp > tcp_full_space(sk) || tp->window_clamp == 0))
3796 		tp->window_clamp = tcp_full_space(sk);
3797 
3798 	rcv_wnd = tcp_rwnd_init_bpf(sk);
3799 	if (rcv_wnd == 0)
3800 		rcv_wnd = dst_metric(dst, RTAX_INITRWND);
3801 
3802 	tcp_select_initial_window(sk, tcp_full_space(sk),
3803 				  tp->advmss - (tp->rx_opt.ts_recent_stamp ? tp->tcp_header_len - sizeof(struct tcphdr) : 0),
3804 				  &tp->rcv_wnd,
3805 				  &tp->window_clamp,
3806 				  READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_window_scaling),
3807 				  &rcv_wscale,
3808 				  rcv_wnd);
3809 
3810 	tp->rx_opt.rcv_wscale = rcv_wscale;
3811 	tp->rcv_ssthresh = tp->rcv_wnd;
3812 
3813 	WRITE_ONCE(sk->sk_err, 0);
3814 	sock_reset_flag(sk, SOCK_DONE);
3815 	tp->snd_wnd = 0;
3816 	tcp_init_wl(tp, 0);
3817 	tcp_write_queue_purge(sk);
3818 	tp->snd_una = tp->write_seq;
3819 	tp->snd_sml = tp->write_seq;
3820 	tp->snd_up = tp->write_seq;
3821 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
3822 
3823 	if (likely(!tp->repair))
3824 		tp->rcv_nxt = 0;
3825 	else
3826 		tp->rcv_tstamp = tcp_jiffies32;
3827 	tp->rcv_wup = tp->rcv_nxt;
3828 	WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3829 
3830 	inet_csk(sk)->icsk_rto = tcp_timeout_init(sk);
3831 	inet_csk(sk)->icsk_retransmits = 0;
3832 	tcp_clear_retrans(tp);
3833 }
3834 
tcp_connect_queue_skb(struct sock * sk,struct sk_buff * skb)3835 static void tcp_connect_queue_skb(struct sock *sk, struct sk_buff *skb)
3836 {
3837 	struct tcp_sock *tp = tcp_sk(sk);
3838 	struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
3839 
3840 	tcb->end_seq += skb->len;
3841 	__skb_header_release(skb);
3842 	sk_wmem_queued_add(sk, skb->truesize);
3843 	sk_mem_charge(sk, skb->truesize);
3844 	WRITE_ONCE(tp->write_seq, tcb->end_seq);
3845 	tp->packets_out += tcp_skb_pcount(skb);
3846 }
3847 
3848 /* Build and send a SYN with data and (cached) Fast Open cookie. However,
3849  * queue a data-only packet after the regular SYN, such that regular SYNs
3850  * are retransmitted on timeouts. Also if the remote SYN-ACK acknowledges
3851  * only the SYN sequence, the data are retransmitted in the first ACK.
3852  * If cookie is not cached or other error occurs, falls back to send a
3853  * regular SYN with Fast Open cookie request option.
3854  */
tcp_send_syn_data(struct sock * sk,struct sk_buff * syn)3855 static int tcp_send_syn_data(struct sock *sk, struct sk_buff *syn)
3856 {
3857 	struct inet_connection_sock *icsk = inet_csk(sk);
3858 	struct tcp_sock *tp = tcp_sk(sk);
3859 	struct tcp_fastopen_request *fo = tp->fastopen_req;
3860 	struct page_frag *pfrag = sk_page_frag(sk);
3861 	struct sk_buff *syn_data;
3862 	int space, err = 0;
3863 
3864 	tp->rx_opt.mss_clamp = tp->advmss;  /* If MSS is not cached */
3865 	if (!tcp_fastopen_cookie_check(sk, &tp->rx_opt.mss_clamp, &fo->cookie))
3866 		goto fallback;
3867 
3868 	/* MSS for SYN-data is based on cached MSS and bounded by PMTU and
3869 	 * user-MSS. Reserve maximum option space for middleboxes that add
3870 	 * private TCP options. The cost is reduced data space in SYN :(
3871 	 */
3872 	tp->rx_opt.mss_clamp = tcp_mss_clamp(tp, tp->rx_opt.mss_clamp);
3873 	/* Sync mss_cache after updating the mss_clamp */
3874 	tcp_sync_mss(sk, icsk->icsk_pmtu_cookie);
3875 
3876 	space = __tcp_mtu_to_mss(sk, icsk->icsk_pmtu_cookie) -
3877 		MAX_TCP_OPTION_SPACE;
3878 
3879 	space = min_t(size_t, space, fo->size);
3880 
3881 	if (space &&
3882 	    !skb_page_frag_refill(min_t(size_t, space, PAGE_SIZE),
3883 				  pfrag, sk->sk_allocation))
3884 		goto fallback;
3885 	syn_data = tcp_stream_alloc_skb(sk, sk->sk_allocation, false);
3886 	if (!syn_data)
3887 		goto fallback;
3888 	memcpy(syn_data->cb, syn->cb, sizeof(syn->cb));
3889 	if (space) {
3890 		space = min_t(size_t, space, pfrag->size - pfrag->offset);
3891 		space = tcp_wmem_schedule(sk, space);
3892 	}
3893 	if (space) {
3894 		space = copy_page_from_iter(pfrag->page, pfrag->offset,
3895 					    space, &fo->data->msg_iter);
3896 		if (unlikely(!space)) {
3897 			tcp_skb_tsorted_anchor_cleanup(syn_data);
3898 			kfree_skb(syn_data);
3899 			goto fallback;
3900 		}
3901 		skb_fill_page_desc(syn_data, 0, pfrag->page,
3902 				   pfrag->offset, space);
3903 		page_ref_inc(pfrag->page);
3904 		pfrag->offset += space;
3905 		skb_len_add(syn_data, space);
3906 		skb_zcopy_set(syn_data, fo->uarg, NULL);
3907 	}
3908 	/* No more data pending in inet_wait_for_connect() */
3909 	if (space == fo->size)
3910 		fo->data = NULL;
3911 	fo->copied = space;
3912 
3913 	tcp_connect_queue_skb(sk, syn_data);
3914 	if (syn_data->len)
3915 		tcp_chrono_start(sk, TCP_CHRONO_BUSY);
3916 
3917 	err = tcp_transmit_skb(sk, syn_data, 1, sk->sk_allocation);
3918 
3919 	skb_set_delivery_time(syn, syn_data->skb_mstamp_ns, true);
3920 
3921 	/* Now full SYN+DATA was cloned and sent (or not),
3922 	 * remove the SYN from the original skb (syn_data)
3923 	 * we keep in write queue in case of a retransmit, as we
3924 	 * also have the SYN packet (with no data) in the same queue.
3925 	 */
3926 	TCP_SKB_CB(syn_data)->seq++;
3927 	TCP_SKB_CB(syn_data)->tcp_flags = TCPHDR_ACK | TCPHDR_PSH;
3928 	if (!err) {
3929 		tp->syn_data = (fo->copied > 0);
3930 		tcp_rbtree_insert(&sk->tcp_rtx_queue, syn_data);
3931 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPORIGDATASENT);
3932 		goto done;
3933 	}
3934 
3935 	/* data was not sent, put it in write_queue */
3936 	__skb_queue_tail(&sk->sk_write_queue, syn_data);
3937 	tp->packets_out -= tcp_skb_pcount(syn_data);
3938 
3939 fallback:
3940 	/* Send a regular SYN with Fast Open cookie request option */
3941 	if (fo->cookie.len > 0)
3942 		fo->cookie.len = 0;
3943 	err = tcp_transmit_skb(sk, syn, 1, sk->sk_allocation);
3944 	if (err)
3945 		tp->syn_fastopen = 0;
3946 done:
3947 	fo->cookie.len = -1;  /* Exclude Fast Open option for SYN retries */
3948 	return err;
3949 }
3950 
3951 /* Build a SYN and send it off. */
tcp_connect(struct sock * sk)3952 int tcp_connect(struct sock *sk)
3953 {
3954 	struct tcp_sock *tp = tcp_sk(sk);
3955 	struct sk_buff *buff;
3956 	int err;
3957 
3958 	tcp_call_bpf(sk, BPF_SOCK_OPS_TCP_CONNECT_CB, 0, NULL);
3959 
3960 	if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk))
3961 		return -EHOSTUNREACH; /* Routing failure or similar. */
3962 
3963 	tcp_connect_init(sk);
3964 
3965 	if (unlikely(tp->repair)) {
3966 		tcp_finish_connect(sk, NULL);
3967 		return 0;
3968 	}
3969 
3970 	buff = tcp_stream_alloc_skb(sk, sk->sk_allocation, true);
3971 	if (unlikely(!buff))
3972 		return -ENOBUFS;
3973 
3974 	tcp_init_nondata_skb(buff, tp->write_seq++, TCPHDR_SYN);
3975 	tcp_mstamp_refresh(tp);
3976 	tp->retrans_stamp = tcp_time_stamp(tp);
3977 	tcp_connect_queue_skb(sk, buff);
3978 	tcp_ecn_send_syn(sk, buff);
3979 	tcp_rbtree_insert(&sk->tcp_rtx_queue, buff);
3980 
3981 	/* Send off SYN; include data in Fast Open. */
3982 	err = tp->fastopen_req ? tcp_send_syn_data(sk, buff) :
3983 	      tcp_transmit_skb(sk, buff, 1, sk->sk_allocation);
3984 	if (err == -ECONNREFUSED)
3985 		return err;
3986 
3987 	/* We change tp->snd_nxt after the tcp_transmit_skb() call
3988 	 * in order to make this packet get counted in tcpOutSegs.
3989 	 */
3990 	WRITE_ONCE(tp->snd_nxt, tp->write_seq);
3991 	tp->pushed_seq = tp->write_seq;
3992 	buff = tcp_send_head(sk);
3993 	if (unlikely(buff)) {
3994 		WRITE_ONCE(tp->snd_nxt, TCP_SKB_CB(buff)->seq);
3995 		tp->pushed_seq	= TCP_SKB_CB(buff)->seq;
3996 	}
3997 	TCP_INC_STATS(sock_net(sk), TCP_MIB_ACTIVEOPENS);
3998 
3999 	/* Timer for repeating the SYN until an answer. */
4000 	inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
4001 				  inet_csk(sk)->icsk_rto, TCP_RTO_MAX);
4002 	return 0;
4003 }
4004 EXPORT_SYMBOL(tcp_connect);
4005 
tcp_delack_max(const struct sock * sk)4006 u32 tcp_delack_max(const struct sock *sk)
4007 {
4008 	const struct dst_entry *dst = __sk_dst_get(sk);
4009 	u32 delack_max = inet_csk(sk)->icsk_delack_max;
4010 
4011 	if (dst && dst_metric_locked(dst, RTAX_RTO_MIN)) {
4012 		u32 rto_min = dst_metric_rtt(dst, RTAX_RTO_MIN);
4013 		u32 delack_from_rto_min = max_t(int, 1, rto_min - 1);
4014 
4015 		delack_max = min_t(u32, delack_max, delack_from_rto_min);
4016 	}
4017 	return delack_max;
4018 }
4019 
4020 /* Send out a delayed ack, the caller does the policy checking
4021  * to see if we should even be here.  See tcp_input.c:tcp_ack_snd_check()
4022  * for details.
4023  */
tcp_send_delayed_ack(struct sock * sk)4024 void tcp_send_delayed_ack(struct sock *sk)
4025 {
4026 	struct inet_connection_sock *icsk = inet_csk(sk);
4027 	int ato = icsk->icsk_ack.ato;
4028 	unsigned long timeout;
4029 
4030 	if (ato > TCP_DELACK_MIN) {
4031 		const struct tcp_sock *tp = tcp_sk(sk);
4032 		int max_ato = HZ / 2;
4033 
4034 		if (inet_csk_in_pingpong_mode(sk) ||
4035 		    (icsk->icsk_ack.pending & ICSK_ACK_PUSHED))
4036 			max_ato = TCP_DELACK_MAX;
4037 
4038 		/* Slow path, intersegment interval is "high". */
4039 
4040 		/* If some rtt estimate is known, use it to bound delayed ack.
4041 		 * Do not use inet_csk(sk)->icsk_rto here, use results of rtt measurements
4042 		 * directly.
4043 		 */
4044 		if (tp->srtt_us) {
4045 			int rtt = max_t(int, usecs_to_jiffies(tp->srtt_us >> 3),
4046 					TCP_DELACK_MIN);
4047 
4048 			if (rtt < max_ato)
4049 				max_ato = rtt;
4050 		}
4051 
4052 		ato = min(ato, max_ato);
4053 	}
4054 
4055 	ato = min_t(u32, ato, tcp_delack_max(sk));
4056 
4057 	/* Stay within the limit we were given */
4058 	timeout = jiffies + ato;
4059 
4060 	/* Use new timeout only if there wasn't a older one earlier. */
4061 	if (icsk->icsk_ack.pending & ICSK_ACK_TIMER) {
4062 		/* If delack timer is about to expire, send ACK now. */
4063 		if (time_before_eq(icsk->icsk_ack.timeout, jiffies + (ato >> 2))) {
4064 			tcp_send_ack(sk);
4065 			return;
4066 		}
4067 
4068 		if (!time_before(timeout, icsk->icsk_ack.timeout))
4069 			timeout = icsk->icsk_ack.timeout;
4070 	}
4071 	icsk->icsk_ack.pending |= ICSK_ACK_SCHED | ICSK_ACK_TIMER;
4072 	icsk->icsk_ack.timeout = timeout;
4073 	sk_reset_timer(sk, &icsk->icsk_delack_timer, timeout);
4074 }
4075 
4076 /* This routine sends an ack and also updates the window. */
__tcp_send_ack(struct sock * sk,u32 rcv_nxt)4077 void __tcp_send_ack(struct sock *sk, u32 rcv_nxt)
4078 {
4079 	struct sk_buff *buff;
4080 
4081 	/* If we have been reset, we may not send again. */
4082 	if (sk->sk_state == TCP_CLOSE)
4083 		return;
4084 
4085 	/* We are not putting this on the write queue, so
4086 	 * tcp_transmit_skb() will set the ownership to this
4087 	 * sock.
4088 	 */
4089 	buff = alloc_skb(MAX_TCP_HEADER,
4090 			 sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4091 	if (unlikely(!buff)) {
4092 		struct inet_connection_sock *icsk = inet_csk(sk);
4093 		unsigned long delay;
4094 
4095 		delay = TCP_DELACK_MAX << icsk->icsk_ack.retry;
4096 		if (delay < TCP_RTO_MAX)
4097 			icsk->icsk_ack.retry++;
4098 		inet_csk_schedule_ack(sk);
4099 		icsk->icsk_ack.ato = TCP_ATO_MIN;
4100 		inet_csk_reset_xmit_timer(sk, ICSK_TIME_DACK, delay, TCP_RTO_MAX);
4101 		return;
4102 	}
4103 
4104 	/* Reserve space for headers and prepare control bits. */
4105 	skb_reserve(buff, MAX_TCP_HEADER);
4106 	tcp_init_nondata_skb(buff, tcp_acceptable_seq(sk), TCPHDR_ACK);
4107 
4108 	/* We do not want pure acks influencing TCP Small Queues or fq/pacing
4109 	 * too much.
4110 	 * SKB_TRUESIZE(max(1 .. 66, MAX_TCP_HEADER)) is unfortunately ~784
4111 	 */
4112 	skb_set_tcp_pure_ack(buff);
4113 
4114 	/* Send it off, this clears delayed acks for us. */
4115 	__tcp_transmit_skb(sk, buff, 0, (__force gfp_t)0, rcv_nxt);
4116 }
4117 EXPORT_SYMBOL_GPL(__tcp_send_ack);
4118 
tcp_send_ack(struct sock * sk)4119 void tcp_send_ack(struct sock *sk)
4120 {
4121 	__tcp_send_ack(sk, tcp_sk(sk)->rcv_nxt);
4122 }
4123 
4124 /* This routine sends a packet with an out of date sequence
4125  * number. It assumes the other end will try to ack it.
4126  *
4127  * Question: what should we make while urgent mode?
4128  * 4.4BSD forces sending single byte of data. We cannot send
4129  * out of window data, because we have SND.NXT==SND.MAX...
4130  *
4131  * Current solution: to send TWO zero-length segments in urgent mode:
4132  * one is with SEG.SEQ=SND.UNA to deliver urgent pointer, another is
4133  * out-of-date with SND.UNA-1 to probe window.
4134  */
tcp_xmit_probe_skb(struct sock * sk,int urgent,int mib)4135 static int tcp_xmit_probe_skb(struct sock *sk, int urgent, int mib)
4136 {
4137 	struct tcp_sock *tp = tcp_sk(sk);
4138 	struct sk_buff *skb;
4139 
4140 	/* We don't queue it, tcp_transmit_skb() sets ownership. */
4141 	skb = alloc_skb(MAX_TCP_HEADER,
4142 			sk_gfp_mask(sk, GFP_ATOMIC | __GFP_NOWARN));
4143 	if (!skb)
4144 		return -1;
4145 
4146 	/* Reserve space for headers and set control bits. */
4147 	skb_reserve(skb, MAX_TCP_HEADER);
4148 	/* Use a previous sequence.  This should cause the other
4149 	 * end to send an ack.  Don't queue or clone SKB, just
4150 	 * send it.
4151 	 */
4152 	tcp_init_nondata_skb(skb, tp->snd_una - !urgent, TCPHDR_ACK);
4153 	NET_INC_STATS(sock_net(sk), mib);
4154 	return tcp_transmit_skb(sk, skb, 0, (__force gfp_t)0);
4155 }
4156 
4157 /* Called from setsockopt( ... TCP_REPAIR ) */
tcp_send_window_probe(struct sock * sk)4158 void tcp_send_window_probe(struct sock *sk)
4159 {
4160 	if (sk->sk_state == TCP_ESTABLISHED) {
4161 		tcp_sk(sk)->snd_wl1 = tcp_sk(sk)->rcv_nxt - 1;
4162 		tcp_mstamp_refresh(tcp_sk(sk));
4163 		tcp_xmit_probe_skb(sk, 0, LINUX_MIB_TCPWINPROBE);
4164 	}
4165 }
4166 
4167 /* Initiate keepalive or window probe from timer. */
tcp_write_wakeup(struct sock * sk,int mib)4168 int tcp_write_wakeup(struct sock *sk, int mib)
4169 {
4170 	struct tcp_sock *tp = tcp_sk(sk);
4171 	struct sk_buff *skb;
4172 
4173 	if (sk->sk_state == TCP_CLOSE)
4174 		return -1;
4175 
4176 	skb = tcp_send_head(sk);
4177 	if (skb && before(TCP_SKB_CB(skb)->seq, tcp_wnd_end(tp))) {
4178 		int err;
4179 		unsigned int mss = tcp_current_mss(sk);
4180 		unsigned int seg_size = tcp_wnd_end(tp) - TCP_SKB_CB(skb)->seq;
4181 
4182 		if (before(tp->pushed_seq, TCP_SKB_CB(skb)->end_seq))
4183 			tp->pushed_seq = TCP_SKB_CB(skb)->end_seq;
4184 
4185 		/* We are probing the opening of a window
4186 		 * but the window size is != 0
4187 		 * must have been a result SWS avoidance ( sender )
4188 		 */
4189 		if (seg_size < TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq ||
4190 		    skb->len > mss) {
4191 			seg_size = min(seg_size, mss);
4192 			TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4193 			if (tcp_fragment(sk, TCP_FRAG_IN_WRITE_QUEUE,
4194 					 skb, seg_size, mss, GFP_ATOMIC))
4195 				return -1;
4196 		} else if (!tcp_skb_pcount(skb))
4197 			tcp_set_skb_tso_segs(skb, mss);
4198 
4199 		TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
4200 		err = tcp_transmit_skb(sk, skb, 1, GFP_ATOMIC);
4201 		if (!err)
4202 			tcp_event_new_data_sent(sk, skb);
4203 		return err;
4204 	} else {
4205 		if (between(tp->snd_up, tp->snd_una + 1, tp->snd_una + 0xFFFF))
4206 			tcp_xmit_probe_skb(sk, 1, mib);
4207 		return tcp_xmit_probe_skb(sk, 0, mib);
4208 	}
4209 }
4210 
4211 /* A window probe timeout has occurred.  If window is not closed send
4212  * a partial packet else a zero probe.
4213  */
tcp_send_probe0(struct sock * sk)4214 void tcp_send_probe0(struct sock *sk)
4215 {
4216 	struct inet_connection_sock *icsk = inet_csk(sk);
4217 	struct tcp_sock *tp = tcp_sk(sk);
4218 	struct net *net = sock_net(sk);
4219 	unsigned long timeout;
4220 	int err;
4221 
4222 	err = tcp_write_wakeup(sk, LINUX_MIB_TCPWINPROBE);
4223 
4224 	if (tp->packets_out || tcp_write_queue_empty(sk)) {
4225 		/* Cancel probe timer, if it is not required. */
4226 		icsk->icsk_probes_out = 0;
4227 		icsk->icsk_backoff = 0;
4228 		icsk->icsk_probes_tstamp = 0;
4229 		return;
4230 	}
4231 
4232 	icsk->icsk_probes_out++;
4233 	if (err <= 0) {
4234 		if (icsk->icsk_backoff < READ_ONCE(net->ipv4.sysctl_tcp_retries2))
4235 			icsk->icsk_backoff++;
4236 		timeout = tcp_probe0_when(sk, TCP_RTO_MAX);
4237 	} else {
4238 		/* If packet was not sent due to local congestion,
4239 		 * Let senders fight for local resources conservatively.
4240 		 */
4241 		timeout = TCP_RESOURCE_PROBE_INTERVAL;
4242 	}
4243 
4244 	timeout = tcp_clamp_probe0_to_user_timeout(sk, timeout);
4245 	tcp_reset_xmit_timer(sk, ICSK_TIME_PROBE0, timeout, TCP_RTO_MAX);
4246 }
4247 
tcp_rtx_synack(const struct sock * sk,struct request_sock * req)4248 int tcp_rtx_synack(const struct sock *sk, struct request_sock *req)
4249 {
4250 	const struct tcp_request_sock_ops *af_ops = tcp_rsk(req)->af_specific;
4251 	struct flowi fl;
4252 	int res;
4253 
4254 	/* Paired with WRITE_ONCE() in sock_setsockopt() */
4255 	if (READ_ONCE(sk->sk_txrehash) == SOCK_TXREHASH_ENABLED)
4256 		WRITE_ONCE(tcp_rsk(req)->txhash, net_tx_rndhash());
4257 	res = af_ops->send_synack(sk, NULL, &fl, req, NULL, TCP_SYNACK_NORMAL,
4258 				  NULL);
4259 	if (!res) {
4260 		TCP_INC_STATS(sock_net(sk), TCP_MIB_RETRANSSEGS);
4261 		NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPSYNRETRANS);
4262 		if (unlikely(tcp_passive_fastopen(sk))) {
4263 			/* sk has const attribute because listeners are lockless.
4264 			 * However in this case, we are dealing with a passive fastopen
4265 			 * socket thus we can change total_retrans value.
4266 			 */
4267 			tcp_sk_rw(sk)->total_retrans++;
4268 		}
4269 		trace_tcp_retransmit_synack(sk, req);
4270 	}
4271 	return res;
4272 }
4273 EXPORT_SYMBOL(tcp_rtx_synack);
4274