1 // SPDX-License-Identifier: GPL-2.0
2 /* Multipath TCP
3  *
4  * Copyright (c) 2017 - 2019, Intel Corporation.
5  */
6 
7 #define pr_fmt(fmt) "MPTCP: " fmt
8 
9 #include <linux/kernel.h>
10 #include <crypto/sha2.h>
11 #include <net/tcp.h>
12 #include <net/mptcp.h>
13 #include "protocol.h"
14 #include "mib.h"
15 
16 #include <trace/events/mptcp.h>
17 
mptcp_cap_flag_sha256(u8 flags)18 static bool mptcp_cap_flag_sha256(u8 flags)
19 {
20 	return (flags & MPTCP_CAP_FLAG_MASK) == MPTCP_CAP_HMAC_SHA256;
21 }
22 
mptcp_parse_option(const struct sk_buff * skb,const unsigned char * ptr,int opsize,struct mptcp_options_received * mp_opt)23 static void mptcp_parse_option(const struct sk_buff *skb,
24 			       const unsigned char *ptr, int opsize,
25 			       struct mptcp_options_received *mp_opt)
26 {
27 	u8 subtype = *ptr >> 4;
28 	int expected_opsize;
29 	u8 version;
30 	u8 flags;
31 	u8 i;
32 
33 	switch (subtype) {
34 	case MPTCPOPT_MP_CAPABLE:
35 		/* strict size checking */
36 		if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
37 			if (skb->len > tcp_hdr(skb)->doff << 2)
38 				expected_opsize = TCPOLEN_MPTCP_MPC_ACK_DATA;
39 			else
40 				expected_opsize = TCPOLEN_MPTCP_MPC_ACK;
41 		} else {
42 			if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_ACK)
43 				expected_opsize = TCPOLEN_MPTCP_MPC_SYNACK;
44 			else
45 				expected_opsize = TCPOLEN_MPTCP_MPC_SYN;
46 		}
47 
48 		/* Cfr RFC 8684 Section 3.3.0:
49 		 * If a checksum is present but its use had
50 		 * not been negotiated in the MP_CAPABLE handshake, the receiver MUST
51 		 * close the subflow with a RST, as it is not behaving as negotiated.
52 		 * If a checksum is not present when its use has been negotiated, the
53 		 * receiver MUST close the subflow with a RST, as it is considered
54 		 * broken
55 		 * We parse even option with mismatching csum presence, so that
56 		 * later in subflow_data_ready we can trigger the reset.
57 		 */
58 		if (opsize != expected_opsize &&
59 		    (expected_opsize != TCPOLEN_MPTCP_MPC_ACK_DATA ||
60 		     opsize != TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM))
61 			break;
62 
63 		/* try to be gentle vs future versions on the initial syn */
64 		version = *ptr++ & MPTCP_VERSION_MASK;
65 		if (opsize != TCPOLEN_MPTCP_MPC_SYN) {
66 			if (version != MPTCP_SUPPORTED_VERSION)
67 				break;
68 		} else if (version < MPTCP_SUPPORTED_VERSION) {
69 			break;
70 		}
71 
72 		flags = *ptr++;
73 		if (!mptcp_cap_flag_sha256(flags) ||
74 		    (flags & MPTCP_CAP_EXTENSIBILITY))
75 			break;
76 
77 		/* RFC 6824, Section 3.1:
78 		 * "For the Checksum Required bit (labeled "A"), if either
79 		 * host requires the use of checksums, checksums MUST be used.
80 		 * In other words, the only way for checksums not to be used
81 		 * is if both hosts in their SYNs set A=0."
82 		 */
83 		if (flags & MPTCP_CAP_CHECKSUM_REQD)
84 			mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
85 
86 		mp_opt->deny_join_id0 = !!(flags & MPTCP_CAP_DENY_JOIN_ID0);
87 
88 		mp_opt->suboptions |= OPTIONS_MPTCP_MPC;
89 		if (opsize >= TCPOLEN_MPTCP_MPC_SYNACK) {
90 			mp_opt->sndr_key = get_unaligned_be64(ptr);
91 			ptr += 8;
92 		}
93 		if (opsize >= TCPOLEN_MPTCP_MPC_ACK) {
94 			mp_opt->rcvr_key = get_unaligned_be64(ptr);
95 			ptr += 8;
96 		}
97 		if (opsize >= TCPOLEN_MPTCP_MPC_ACK_DATA) {
98 			/* Section 3.1.:
99 			 * "the data parameters in a MP_CAPABLE are semantically
100 			 * equivalent to those in a DSS option and can be used
101 			 * interchangeably."
102 			 */
103 			mp_opt->suboptions |= OPTION_MPTCP_DSS;
104 			mp_opt->use_map = 1;
105 			mp_opt->mpc_map = 1;
106 			mp_opt->data_len = get_unaligned_be16(ptr);
107 			ptr += 2;
108 		}
109 		if (opsize == TCPOLEN_MPTCP_MPC_ACK_DATA_CSUM) {
110 			mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
111 			mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
112 			ptr += 2;
113 		}
114 		pr_debug("MP_CAPABLE version=%x, flags=%x, optlen=%d sndr=%llu, rcvr=%llu len=%d csum=%u",
115 			 version, flags, opsize, mp_opt->sndr_key,
116 			 mp_opt->rcvr_key, mp_opt->data_len, mp_opt->csum);
117 		break;
118 
119 	case MPTCPOPT_MP_JOIN:
120 		mp_opt->suboptions |= OPTIONS_MPTCP_MPJ;
121 		if (opsize == TCPOLEN_MPTCP_MPJ_SYN) {
122 			mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
123 			mp_opt->join_id = *ptr++;
124 			mp_opt->token = get_unaligned_be32(ptr);
125 			ptr += 4;
126 			mp_opt->nonce = get_unaligned_be32(ptr);
127 			ptr += 4;
128 			pr_debug("MP_JOIN bkup=%u, id=%u, token=%u, nonce=%u",
129 				 mp_opt->backup, mp_opt->join_id,
130 				 mp_opt->token, mp_opt->nonce);
131 		} else if (opsize == TCPOLEN_MPTCP_MPJ_SYNACK) {
132 			mp_opt->backup = *ptr++ & MPTCPOPT_BACKUP;
133 			mp_opt->join_id = *ptr++;
134 			mp_opt->thmac = get_unaligned_be64(ptr);
135 			ptr += 8;
136 			mp_opt->nonce = get_unaligned_be32(ptr);
137 			ptr += 4;
138 			pr_debug("MP_JOIN bkup=%u, id=%u, thmac=%llu, nonce=%u",
139 				 mp_opt->backup, mp_opt->join_id,
140 				 mp_opt->thmac, mp_opt->nonce);
141 		} else if (opsize == TCPOLEN_MPTCP_MPJ_ACK) {
142 			ptr += 2;
143 			memcpy(mp_opt->hmac, ptr, MPTCPOPT_HMAC_LEN);
144 			pr_debug("MP_JOIN hmac");
145 		} else {
146 			mp_opt->suboptions &= ~OPTIONS_MPTCP_MPJ;
147 		}
148 		break;
149 
150 	case MPTCPOPT_DSS:
151 		pr_debug("DSS");
152 		ptr++;
153 
154 		/* we must clear 'mpc_map' be able to detect MP_CAPABLE
155 		 * map vs DSS map in mptcp_incoming_options(), and reconstruct
156 		 * map info accordingly
157 		 */
158 		mp_opt->mpc_map = 0;
159 		flags = (*ptr++) & MPTCP_DSS_FLAG_MASK;
160 		mp_opt->data_fin = (flags & MPTCP_DSS_DATA_FIN) != 0;
161 		mp_opt->dsn64 = (flags & MPTCP_DSS_DSN64) != 0;
162 		mp_opt->use_map = (flags & MPTCP_DSS_HAS_MAP) != 0;
163 		mp_opt->ack64 = (flags & MPTCP_DSS_ACK64) != 0;
164 		mp_opt->use_ack = (flags & MPTCP_DSS_HAS_ACK);
165 
166 		pr_debug("data_fin=%d dsn64=%d use_map=%d ack64=%d use_ack=%d",
167 			 mp_opt->data_fin, mp_opt->dsn64,
168 			 mp_opt->use_map, mp_opt->ack64,
169 			 mp_opt->use_ack);
170 
171 		expected_opsize = TCPOLEN_MPTCP_DSS_BASE;
172 
173 		if (mp_opt->use_ack) {
174 			if (mp_opt->ack64)
175 				expected_opsize += TCPOLEN_MPTCP_DSS_ACK64;
176 			else
177 				expected_opsize += TCPOLEN_MPTCP_DSS_ACK32;
178 		}
179 
180 		if (mp_opt->use_map) {
181 			if (mp_opt->dsn64)
182 				expected_opsize += TCPOLEN_MPTCP_DSS_MAP64;
183 			else
184 				expected_opsize += TCPOLEN_MPTCP_DSS_MAP32;
185 		}
186 
187 		/* Always parse any csum presence combination, we will enforce
188 		 * RFC 8684 Section 3.3.0 checks later in subflow_data_ready
189 		 */
190 		if (opsize != expected_opsize &&
191 		    opsize != expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM)
192 			break;
193 
194 		mp_opt->suboptions |= OPTION_MPTCP_DSS;
195 		if (mp_opt->use_ack) {
196 			if (mp_opt->ack64) {
197 				mp_opt->data_ack = get_unaligned_be64(ptr);
198 				ptr += 8;
199 			} else {
200 				mp_opt->data_ack = get_unaligned_be32(ptr);
201 				ptr += 4;
202 			}
203 
204 			pr_debug("data_ack=%llu", mp_opt->data_ack);
205 		}
206 
207 		if (mp_opt->use_map) {
208 			if (mp_opt->dsn64) {
209 				mp_opt->data_seq = get_unaligned_be64(ptr);
210 				ptr += 8;
211 			} else {
212 				mp_opt->data_seq = get_unaligned_be32(ptr);
213 				ptr += 4;
214 			}
215 
216 			mp_opt->subflow_seq = get_unaligned_be32(ptr);
217 			ptr += 4;
218 
219 			mp_opt->data_len = get_unaligned_be16(ptr);
220 			ptr += 2;
221 
222 			if (opsize == expected_opsize + TCPOLEN_MPTCP_DSS_CHECKSUM) {
223 				mp_opt->suboptions |= OPTION_MPTCP_CSUMREQD;
224 				mp_opt->csum = get_unaligned((__force __sum16 *)ptr);
225 				ptr += 2;
226 			}
227 
228 			pr_debug("data_seq=%llu subflow_seq=%u data_len=%u csum=%d:%u",
229 				 mp_opt->data_seq, mp_opt->subflow_seq,
230 				 mp_opt->data_len, !!(mp_opt->suboptions & OPTION_MPTCP_CSUMREQD),
231 				 mp_opt->csum);
232 		}
233 
234 		break;
235 
236 	case MPTCPOPT_ADD_ADDR:
237 		mp_opt->echo = (*ptr++) & MPTCP_ADDR_ECHO;
238 		if (!mp_opt->echo) {
239 			if (opsize == TCPOLEN_MPTCP_ADD_ADDR ||
240 			    opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT)
241 				mp_opt->addr.family = AF_INET;
242 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
243 			else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6 ||
244 				 opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT)
245 				mp_opt->addr.family = AF_INET6;
246 #endif
247 			else
248 				break;
249 		} else {
250 			if (opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE ||
251 			    opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT)
252 				mp_opt->addr.family = AF_INET;
253 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
254 			else if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE ||
255 				 opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT)
256 				mp_opt->addr.family = AF_INET6;
257 #endif
258 			else
259 				break;
260 		}
261 
262 		mp_opt->suboptions |= OPTION_MPTCP_ADD_ADDR;
263 		mp_opt->addr.id = *ptr++;
264 		mp_opt->addr.port = 0;
265 		mp_opt->ahmac = 0;
266 		if (mp_opt->addr.family == AF_INET) {
267 			memcpy((u8 *)&mp_opt->addr.addr.s_addr, (u8 *)ptr, 4);
268 			ptr += 4;
269 			if (opsize == TCPOLEN_MPTCP_ADD_ADDR_PORT ||
270 			    opsize == TCPOLEN_MPTCP_ADD_ADDR_BASE_PORT) {
271 				mp_opt->addr.port = htons(get_unaligned_be16(ptr));
272 				ptr += 2;
273 			}
274 		}
275 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
276 		else {
277 			memcpy(mp_opt->addr.addr6.s6_addr, (u8 *)ptr, 16);
278 			ptr += 16;
279 			if (opsize == TCPOLEN_MPTCP_ADD_ADDR6_PORT ||
280 			    opsize == TCPOLEN_MPTCP_ADD_ADDR6_BASE_PORT) {
281 				mp_opt->addr.port = htons(get_unaligned_be16(ptr));
282 				ptr += 2;
283 			}
284 		}
285 #endif
286 		if (!mp_opt->echo) {
287 			mp_opt->ahmac = get_unaligned_be64(ptr);
288 			ptr += 8;
289 		}
290 		pr_debug("ADD_ADDR%s: id=%d, ahmac=%llu, echo=%d, port=%d",
291 			 (mp_opt->addr.family == AF_INET6) ? "6" : "",
292 			 mp_opt->addr.id, mp_opt->ahmac, mp_opt->echo, ntohs(mp_opt->addr.port));
293 		break;
294 
295 	case MPTCPOPT_RM_ADDR:
296 		if (opsize < TCPOLEN_MPTCP_RM_ADDR_BASE + 1 ||
297 		    opsize > TCPOLEN_MPTCP_RM_ADDR_BASE + MPTCP_RM_IDS_MAX)
298 			break;
299 
300 		ptr++;
301 
302 		mp_opt->suboptions |= OPTION_MPTCP_RM_ADDR;
303 		mp_opt->rm_list.nr = opsize - TCPOLEN_MPTCP_RM_ADDR_BASE;
304 		for (i = 0; i < mp_opt->rm_list.nr; i++)
305 			mp_opt->rm_list.ids[i] = *ptr++;
306 		pr_debug("RM_ADDR: rm_list_nr=%d", mp_opt->rm_list.nr);
307 		break;
308 
309 	case MPTCPOPT_MP_PRIO:
310 		if (opsize != TCPOLEN_MPTCP_PRIO)
311 			break;
312 
313 		mp_opt->suboptions |= OPTION_MPTCP_PRIO;
314 		mp_opt->backup = *ptr++ & MPTCP_PRIO_BKUP;
315 		pr_debug("MP_PRIO: prio=%d", mp_opt->backup);
316 		break;
317 
318 	case MPTCPOPT_MP_FASTCLOSE:
319 		if (opsize != TCPOLEN_MPTCP_FASTCLOSE)
320 			break;
321 
322 		ptr += 2;
323 		mp_opt->rcvr_key = get_unaligned_be64(ptr);
324 		ptr += 8;
325 		mp_opt->suboptions |= OPTION_MPTCP_FASTCLOSE;
326 		pr_debug("MP_FASTCLOSE: recv_key=%llu", mp_opt->rcvr_key);
327 		break;
328 
329 	case MPTCPOPT_RST:
330 		if (opsize != TCPOLEN_MPTCP_RST)
331 			break;
332 
333 		if (!(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST))
334 			break;
335 
336 		mp_opt->suboptions |= OPTION_MPTCP_RST;
337 		flags = *ptr++;
338 		mp_opt->reset_transient = flags & MPTCP_RST_TRANSIENT;
339 		mp_opt->reset_reason = *ptr;
340 		pr_debug("MP_RST: transient=%u reason=%u",
341 			 mp_opt->reset_transient, mp_opt->reset_reason);
342 		break;
343 
344 	case MPTCPOPT_MP_FAIL:
345 		if (opsize != TCPOLEN_MPTCP_FAIL)
346 			break;
347 
348 		ptr += 2;
349 		mp_opt->suboptions |= OPTION_MPTCP_FAIL;
350 		mp_opt->fail_seq = get_unaligned_be64(ptr);
351 		pr_debug("MP_FAIL: data_seq=%llu", mp_opt->fail_seq);
352 		break;
353 
354 	default:
355 		break;
356 	}
357 }
358 
mptcp_get_options(const struct sk_buff * skb,struct mptcp_options_received * mp_opt)359 void mptcp_get_options(const struct sk_buff *skb,
360 		       struct mptcp_options_received *mp_opt)
361 {
362 	const struct tcphdr *th = tcp_hdr(skb);
363 	const unsigned char *ptr;
364 	int length;
365 
366 	/* initialize option status */
367 	mp_opt->suboptions = 0;
368 
369 	length = (th->doff * 4) - sizeof(struct tcphdr);
370 	ptr = (const unsigned char *)(th + 1);
371 
372 	while (length > 0) {
373 		int opcode = *ptr++;
374 		int opsize;
375 
376 		switch (opcode) {
377 		case TCPOPT_EOL:
378 			return;
379 		case TCPOPT_NOP:	/* Ref: RFC 793 section 3.1 */
380 			length--;
381 			continue;
382 		default:
383 			if (length < 2)
384 				return;
385 			opsize = *ptr++;
386 			if (opsize < 2) /* "silly options" */
387 				return;
388 			if (opsize > length)
389 				return;	/* don't parse partial options */
390 			if (opcode == TCPOPT_MPTCP)
391 				mptcp_parse_option(skb, ptr, opsize, mp_opt);
392 			ptr += opsize - 2;
393 			length -= opsize;
394 		}
395 	}
396 }
397 
mptcp_syn_options(struct sock * sk,const struct sk_buff * skb,unsigned int * size,struct mptcp_out_options * opts)398 bool mptcp_syn_options(struct sock *sk, const struct sk_buff *skb,
399 		       unsigned int *size, struct mptcp_out_options *opts)
400 {
401 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
402 
403 	/* we will use snd_isn to detect first pkt [re]transmission
404 	 * in mptcp_established_options_mp()
405 	 */
406 	subflow->snd_isn = TCP_SKB_CB(skb)->end_seq;
407 	if (subflow->request_mptcp) {
408 		opts->suboptions = OPTION_MPTCP_MPC_SYN;
409 		opts->csum_reqd = mptcp_is_checksum_enabled(sock_net(sk));
410 		opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
411 		*size = TCPOLEN_MPTCP_MPC_SYN;
412 		return true;
413 	} else if (subflow->request_join) {
414 		pr_debug("remote_token=%u, nonce=%u", subflow->remote_token,
415 			 subflow->local_nonce);
416 		opts->suboptions = OPTION_MPTCP_MPJ_SYN;
417 		opts->join_id = subflow->local_id;
418 		opts->token = subflow->remote_token;
419 		opts->nonce = subflow->local_nonce;
420 		opts->backup = subflow->request_bkup;
421 		*size = TCPOLEN_MPTCP_MPJ_SYN;
422 		return true;
423 	}
424 	return false;
425 }
426 
clear_3rdack_retransmission(struct sock * sk)427 static void clear_3rdack_retransmission(struct sock *sk)
428 {
429 	struct inet_connection_sock *icsk = inet_csk(sk);
430 
431 	sk_stop_timer(sk, &icsk->icsk_delack_timer);
432 	icsk->icsk_ack.timeout = 0;
433 	icsk->icsk_ack.ato = 0;
434 	icsk->icsk_ack.pending &= ~(ICSK_ACK_SCHED | ICSK_ACK_TIMER);
435 }
436 
mptcp_established_options_mp(struct sock * sk,struct sk_buff * skb,bool snd_data_fin_enable,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)437 static bool mptcp_established_options_mp(struct sock *sk, struct sk_buff *skb,
438 					 bool snd_data_fin_enable,
439 					 unsigned int *size,
440 					 unsigned int remaining,
441 					 struct mptcp_out_options *opts)
442 {
443 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
444 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
445 	struct mptcp_ext *mpext;
446 	unsigned int data_len;
447 	u8 len;
448 
449 	/* When skb is not available, we better over-estimate the emitted
450 	 * options len. A full DSS option (28 bytes) is longer than
451 	 * TCPOLEN_MPTCP_MPC_ACK_DATA(22) or TCPOLEN_MPTCP_MPJ_ACK(24), so
452 	 * tell the caller to defer the estimate to
453 	 * mptcp_established_options_dss(), which will reserve enough space.
454 	 */
455 	if (!skb)
456 		return false;
457 
458 	/* MPC/MPJ needed only on 3rd ack packet, DATA_FIN and TCP shutdown take precedence */
459 	if (subflow->fully_established || snd_data_fin_enable ||
460 	    subflow->snd_isn != TCP_SKB_CB(skb)->seq ||
461 	    sk->sk_state != TCP_ESTABLISHED)
462 		return false;
463 
464 	if (subflow->mp_capable) {
465 		mpext = mptcp_get_ext(skb);
466 		data_len = mpext ? mpext->data_len : 0;
467 
468 		/* we will check ops->data_len in mptcp_write_options() to
469 		 * discriminate between TCPOLEN_MPTCP_MPC_ACK_DATA and
470 		 * TCPOLEN_MPTCP_MPC_ACK
471 		 */
472 		opts->data_len = data_len;
473 		opts->suboptions = OPTION_MPTCP_MPC_ACK;
474 		opts->sndr_key = subflow->local_key;
475 		opts->rcvr_key = subflow->remote_key;
476 		opts->csum_reqd = READ_ONCE(msk->csum_enabled);
477 		opts->allow_join_id0 = mptcp_allow_join_id0(sock_net(sk));
478 
479 		/* Section 3.1.
480 		 * The MP_CAPABLE option is carried on the SYN, SYN/ACK, and ACK
481 		 * packets that start the first subflow of an MPTCP connection,
482 		 * as well as the first packet that carries data
483 		 */
484 		if (data_len > 0) {
485 			len = TCPOLEN_MPTCP_MPC_ACK_DATA;
486 			if (opts->csum_reqd) {
487 				/* we need to propagate more info to csum the pseudo hdr */
488 				opts->data_seq = mpext->data_seq;
489 				opts->subflow_seq = mpext->subflow_seq;
490 				opts->csum = mpext->csum;
491 				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
492 			}
493 			*size = ALIGN(len, 4);
494 		} else {
495 			*size = TCPOLEN_MPTCP_MPC_ACK;
496 		}
497 
498 		pr_debug("subflow=%p, local_key=%llu, remote_key=%llu map_len=%d",
499 			 subflow, subflow->local_key, subflow->remote_key,
500 			 data_len);
501 
502 		return true;
503 	} else if (subflow->mp_join) {
504 		opts->suboptions = OPTION_MPTCP_MPJ_ACK;
505 		memcpy(opts->hmac, subflow->hmac, MPTCPOPT_HMAC_LEN);
506 		*size = TCPOLEN_MPTCP_MPJ_ACK;
507 		pr_debug("subflow=%p", subflow);
508 
509 		/* we can use the full delegate action helper only from BH context
510 		 * If we are in process context - sk is flushing the backlog at
511 		 * socket lock release time - just set the appropriate flag, will
512 		 * be handled by the release callback
513 		 */
514 		if (sock_owned_by_user(sk))
515 			set_bit(MPTCP_DELEGATE_ACK, &subflow->delegated_status);
516 		else
517 			mptcp_subflow_delegate(subflow, MPTCP_DELEGATE_ACK);
518 		return true;
519 	}
520 	return false;
521 }
522 
mptcp_write_data_fin(struct mptcp_subflow_context * subflow,struct sk_buff * skb,struct mptcp_ext * ext)523 static void mptcp_write_data_fin(struct mptcp_subflow_context *subflow,
524 				 struct sk_buff *skb, struct mptcp_ext *ext)
525 {
526 	/* The write_seq value has already been incremented, so the actual
527 	 * sequence number for the DATA_FIN is one less.
528 	 */
529 	u64 data_fin_tx_seq = READ_ONCE(mptcp_sk(subflow->conn)->write_seq) - 1;
530 
531 	if (!ext->use_map || !skb->len) {
532 		/* RFC6824 requires a DSS mapping with specific values
533 		 * if DATA_FIN is set but no data payload is mapped
534 		 */
535 		ext->data_fin = 1;
536 		ext->use_map = 1;
537 		ext->dsn64 = 1;
538 		ext->data_seq = data_fin_tx_seq;
539 		ext->subflow_seq = 0;
540 		ext->data_len = 1;
541 	} else if (ext->data_seq + ext->data_len == data_fin_tx_seq) {
542 		/* If there's an existing DSS mapping and it is the
543 		 * final mapping, DATA_FIN consumes 1 additional byte of
544 		 * mapping space.
545 		 */
546 		ext->data_fin = 1;
547 		ext->data_len++;
548 	}
549 }
550 
mptcp_established_options_dss(struct sock * sk,struct sk_buff * skb,bool snd_data_fin_enable,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)551 static bool mptcp_established_options_dss(struct sock *sk, struct sk_buff *skb,
552 					  bool snd_data_fin_enable,
553 					  unsigned int *size,
554 					  unsigned int remaining,
555 					  struct mptcp_out_options *opts)
556 {
557 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
558 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
559 	unsigned int dss_size = 0;
560 	struct mptcp_ext *mpext;
561 	unsigned int ack_size;
562 	bool ret = false;
563 	u64 ack_seq;
564 
565 	opts->csum_reqd = READ_ONCE(msk->csum_enabled);
566 	mpext = skb ? mptcp_get_ext(skb) : NULL;
567 
568 	if (!skb || (mpext && mpext->use_map) || snd_data_fin_enable) {
569 		unsigned int map_size = TCPOLEN_MPTCP_DSS_BASE + TCPOLEN_MPTCP_DSS_MAP64;
570 
571 		if (mpext) {
572 			if (opts->csum_reqd)
573 				map_size += TCPOLEN_MPTCP_DSS_CHECKSUM;
574 
575 			opts->ext_copy = *mpext;
576 		}
577 
578 		remaining -= map_size;
579 		dss_size = map_size;
580 		if (skb && snd_data_fin_enable)
581 			mptcp_write_data_fin(subflow, skb, &opts->ext_copy);
582 		opts->suboptions = OPTION_MPTCP_DSS;
583 		ret = true;
584 	}
585 
586 	/* passive sockets msk will set the 'can_ack' after accept(), even
587 	 * if the first subflow may have the already the remote key handy
588 	 */
589 	opts->ext_copy.use_ack = 0;
590 	if (!READ_ONCE(msk->can_ack)) {
591 		*size = ALIGN(dss_size, 4);
592 		return ret;
593 	}
594 
595 	ack_seq = READ_ONCE(msk->ack_seq);
596 	if (READ_ONCE(msk->use_64bit_ack)) {
597 		ack_size = TCPOLEN_MPTCP_DSS_ACK64;
598 		opts->ext_copy.data_ack = ack_seq;
599 		opts->ext_copy.ack64 = 1;
600 	} else {
601 		ack_size = TCPOLEN_MPTCP_DSS_ACK32;
602 		opts->ext_copy.data_ack32 = (uint32_t)ack_seq;
603 		opts->ext_copy.ack64 = 0;
604 	}
605 	opts->ext_copy.use_ack = 1;
606 	opts->suboptions = OPTION_MPTCP_DSS;
607 	WRITE_ONCE(msk->old_wspace, __mptcp_space((struct sock *)msk));
608 
609 	/* Add kind/length/subtype/flag overhead if mapping is not populated */
610 	if (dss_size == 0)
611 		ack_size += TCPOLEN_MPTCP_DSS_BASE;
612 
613 	dss_size += ack_size;
614 
615 	*size = ALIGN(dss_size, 4);
616 	return true;
617 }
618 
add_addr_generate_hmac(u64 key1,u64 key2,struct mptcp_addr_info * addr)619 static u64 add_addr_generate_hmac(u64 key1, u64 key2,
620 				  struct mptcp_addr_info *addr)
621 {
622 	u16 port = ntohs(addr->port);
623 	u8 hmac[SHA256_DIGEST_SIZE];
624 	u8 msg[19];
625 	int i = 0;
626 
627 	msg[i++] = addr->id;
628 	if (addr->family == AF_INET) {
629 		memcpy(&msg[i], &addr->addr.s_addr, 4);
630 		i += 4;
631 	}
632 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
633 	else if (addr->family == AF_INET6) {
634 		memcpy(&msg[i], &addr->addr6.s6_addr, 16);
635 		i += 16;
636 	}
637 #endif
638 	msg[i++] = port >> 8;
639 	msg[i++] = port & 0xFF;
640 
641 	mptcp_crypto_hmac_sha(key1, key2, msg, i, hmac);
642 
643 	return get_unaligned_be64(&hmac[SHA256_DIGEST_SIZE - sizeof(u64)]);
644 }
645 
mptcp_established_options_add_addr(struct sock * sk,struct sk_buff * skb,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)646 static bool mptcp_established_options_add_addr(struct sock *sk, struct sk_buff *skb,
647 					       unsigned int *size,
648 					       unsigned int remaining,
649 					       struct mptcp_out_options *opts)
650 {
651 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
652 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
653 	bool drop_other_suboptions = false;
654 	unsigned int opt_size = *size;
655 	bool echo;
656 	int len;
657 
658 	/* add addr will strip the existing options, be sure to avoid breaking
659 	 * MPC/MPJ handshakes
660 	 */
661 	if (!mptcp_pm_should_add_signal(msk) ||
662 	    (opts->suboptions & (OPTION_MPTCP_MPJ_ACK | OPTION_MPTCP_MPC_ACK)) ||
663 	    !mptcp_pm_add_addr_signal(msk, skb, opt_size, remaining, &opts->addr,
664 		    &echo, &drop_other_suboptions))
665 		return false;
666 
667 	if (drop_other_suboptions)
668 		remaining += opt_size;
669 	len = mptcp_add_addr_len(opts->addr.family, echo, !!opts->addr.port);
670 	if (remaining < len)
671 		return false;
672 
673 	*size = len;
674 	if (drop_other_suboptions) {
675 		pr_debug("drop other suboptions");
676 		opts->suboptions = 0;
677 
678 		/* note that e.g. DSS could have written into the memory
679 		 * aliased by ahmac, we must reset the field here
680 		 * to avoid appending the hmac even for ADD_ADDR echo
681 		 * options
682 		 */
683 		opts->ahmac = 0;
684 		*size -= opt_size;
685 	}
686 	opts->suboptions |= OPTION_MPTCP_ADD_ADDR;
687 	if (!echo) {
688 		opts->ahmac = add_addr_generate_hmac(msk->local_key,
689 						     msk->remote_key,
690 						     &opts->addr);
691 	}
692 	pr_debug("addr_id=%d, ahmac=%llu, echo=%d, port=%d",
693 		 opts->addr.id, opts->ahmac, echo, ntohs(opts->addr.port));
694 
695 	return true;
696 }
697 
mptcp_established_options_rm_addr(struct sock * sk,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)698 static bool mptcp_established_options_rm_addr(struct sock *sk,
699 					      unsigned int *size,
700 					      unsigned int remaining,
701 					      struct mptcp_out_options *opts)
702 {
703 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
704 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
705 	struct mptcp_rm_list rm_list;
706 	int i, len;
707 
708 	if (!mptcp_pm_should_rm_signal(msk) ||
709 	    !(mptcp_pm_rm_addr_signal(msk, remaining, &rm_list)))
710 		return false;
711 
712 	len = mptcp_rm_addr_len(&rm_list);
713 	if (len < 0)
714 		return false;
715 	if (remaining < len)
716 		return false;
717 
718 	*size = len;
719 	opts->suboptions |= OPTION_MPTCP_RM_ADDR;
720 	opts->rm_list = rm_list;
721 
722 	for (i = 0; i < opts->rm_list.nr; i++)
723 		pr_debug("rm_list_ids[%d]=%d", i, opts->rm_list.ids[i]);
724 
725 	return true;
726 }
727 
mptcp_established_options_mp_prio(struct sock * sk,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)728 static bool mptcp_established_options_mp_prio(struct sock *sk,
729 					      unsigned int *size,
730 					      unsigned int remaining,
731 					      struct mptcp_out_options *opts)
732 {
733 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
734 
735 	/* can't send MP_PRIO with MPC, as they share the same option space:
736 	 * 'backup'. Also it makes no sense at all
737 	 */
738 	if (!subflow->send_mp_prio || (opts->suboptions & OPTIONS_MPTCP_MPC))
739 		return false;
740 
741 	/* account for the trailing 'nop' option */
742 	if (remaining < TCPOLEN_MPTCP_PRIO_ALIGN)
743 		return false;
744 
745 	*size = TCPOLEN_MPTCP_PRIO_ALIGN;
746 	opts->suboptions |= OPTION_MPTCP_PRIO;
747 	opts->backup = subflow->request_bkup;
748 
749 	pr_debug("prio=%d", opts->backup);
750 
751 	return true;
752 }
753 
mptcp_established_options_rst(struct sock * sk,struct sk_buff * skb,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)754 static noinline bool mptcp_established_options_rst(struct sock *sk, struct sk_buff *skb,
755 						   unsigned int *size,
756 						   unsigned int remaining,
757 						   struct mptcp_out_options *opts)
758 {
759 	const struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
760 
761 	if (remaining < TCPOLEN_MPTCP_RST)
762 		return false;
763 
764 	*size = TCPOLEN_MPTCP_RST;
765 	opts->suboptions |= OPTION_MPTCP_RST;
766 	opts->reset_transient = subflow->reset_transient;
767 	opts->reset_reason = subflow->reset_reason;
768 	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTTX);
769 
770 	return true;
771 }
772 
mptcp_established_options_fastclose(struct sock * sk,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)773 static bool mptcp_established_options_fastclose(struct sock *sk,
774 						unsigned int *size,
775 						unsigned int remaining,
776 						struct mptcp_out_options *opts)
777 {
778 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
779 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
780 
781 	if (likely(!subflow->send_fastclose))
782 		return false;
783 
784 	if (remaining < TCPOLEN_MPTCP_FASTCLOSE)
785 		return false;
786 
787 	*size = TCPOLEN_MPTCP_FASTCLOSE;
788 	opts->suboptions |= OPTION_MPTCP_FASTCLOSE;
789 	opts->rcvr_key = msk->remote_key;
790 
791 	pr_debug("FASTCLOSE key=%llu", opts->rcvr_key);
792 	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSETX);
793 	return true;
794 }
795 
mptcp_established_options_mp_fail(struct sock * sk,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)796 static bool mptcp_established_options_mp_fail(struct sock *sk,
797 					      unsigned int *size,
798 					      unsigned int remaining,
799 					      struct mptcp_out_options *opts)
800 {
801 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
802 
803 	if (likely(!subflow->send_mp_fail))
804 		return false;
805 
806 	if (remaining < TCPOLEN_MPTCP_FAIL)
807 		return false;
808 
809 	*size = TCPOLEN_MPTCP_FAIL;
810 	opts->suboptions |= OPTION_MPTCP_FAIL;
811 	opts->fail_seq = subflow->map_seq;
812 
813 	pr_debug("MP_FAIL fail_seq=%llu", opts->fail_seq);
814 	MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILTX);
815 
816 	return true;
817 }
818 
mptcp_established_options(struct sock * sk,struct sk_buff * skb,unsigned int * size,unsigned int remaining,struct mptcp_out_options * opts)819 bool mptcp_established_options(struct sock *sk, struct sk_buff *skb,
820 			       unsigned int *size, unsigned int remaining,
821 			       struct mptcp_out_options *opts)
822 {
823 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
824 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
825 	unsigned int opt_size = 0;
826 	bool snd_data_fin;
827 	bool ret = false;
828 
829 	opts->suboptions = 0;
830 
831 	if (unlikely(__mptcp_check_fallback(msk) && !mptcp_check_infinite_map(skb)))
832 		return false;
833 
834 	if (unlikely(skb && TCP_SKB_CB(skb)->tcp_flags & TCPHDR_RST)) {
835 		if (mptcp_established_options_fastclose(sk, &opt_size, remaining, opts) ||
836 		    mptcp_established_options_mp_fail(sk, &opt_size, remaining, opts)) {
837 			*size += opt_size;
838 			remaining -= opt_size;
839 		}
840 		/* MP_RST can be used with MP_FASTCLOSE and MP_FAIL if there is room */
841 		if (mptcp_established_options_rst(sk, skb, &opt_size, remaining, opts)) {
842 			*size += opt_size;
843 			remaining -= opt_size;
844 		}
845 		return true;
846 	}
847 
848 	snd_data_fin = mptcp_data_fin_enabled(msk);
849 	if (mptcp_established_options_mp(sk, skb, snd_data_fin, &opt_size, remaining, opts))
850 		ret = true;
851 	else if (mptcp_established_options_dss(sk, skb, snd_data_fin, &opt_size, remaining, opts)) {
852 		unsigned int mp_fail_size;
853 
854 		ret = true;
855 		if (mptcp_established_options_mp_fail(sk, &mp_fail_size,
856 						      remaining - opt_size, opts)) {
857 			*size += opt_size + mp_fail_size;
858 			remaining -= opt_size - mp_fail_size;
859 			return true;
860 		}
861 	}
862 
863 	/* we reserved enough space for the above options, and exceeding the
864 	 * TCP option space would be fatal
865 	 */
866 	if (WARN_ON_ONCE(opt_size > remaining))
867 		return false;
868 
869 	*size += opt_size;
870 	remaining -= opt_size;
871 	if (mptcp_established_options_add_addr(sk, skb, &opt_size, remaining, opts)) {
872 		*size += opt_size;
873 		remaining -= opt_size;
874 		ret = true;
875 	} else if (mptcp_established_options_rm_addr(sk, &opt_size, remaining, opts)) {
876 		*size += opt_size;
877 		remaining -= opt_size;
878 		ret = true;
879 	}
880 
881 	if (mptcp_established_options_mp_prio(sk, &opt_size, remaining, opts)) {
882 		*size += opt_size;
883 		remaining -= opt_size;
884 		ret = true;
885 	}
886 
887 	return ret;
888 }
889 
mptcp_synack_options(const struct request_sock * req,unsigned int * size,struct mptcp_out_options * opts)890 bool mptcp_synack_options(const struct request_sock *req, unsigned int *size,
891 			  struct mptcp_out_options *opts)
892 {
893 	struct mptcp_subflow_request_sock *subflow_req = mptcp_subflow_rsk(req);
894 
895 	if (subflow_req->mp_capable) {
896 		opts->suboptions = OPTION_MPTCP_MPC_SYNACK;
897 		opts->sndr_key = subflow_req->local_key;
898 		opts->csum_reqd = subflow_req->csum_reqd;
899 		opts->allow_join_id0 = subflow_req->allow_join_id0;
900 		*size = TCPOLEN_MPTCP_MPC_SYNACK;
901 		pr_debug("subflow_req=%p, local_key=%llu",
902 			 subflow_req, subflow_req->local_key);
903 		return true;
904 	} else if (subflow_req->mp_join) {
905 		opts->suboptions = OPTION_MPTCP_MPJ_SYNACK;
906 		opts->backup = subflow_req->backup;
907 		opts->join_id = subflow_req->local_id;
908 		opts->thmac = subflow_req->thmac;
909 		opts->nonce = subflow_req->local_nonce;
910 		pr_debug("req=%p, bkup=%u, id=%u, thmac=%llu, nonce=%u",
911 			 subflow_req, opts->backup, opts->join_id,
912 			 opts->thmac, opts->nonce);
913 		*size = TCPOLEN_MPTCP_MPJ_SYNACK;
914 		return true;
915 	}
916 	return false;
917 }
918 
check_fully_established(struct mptcp_sock * msk,struct sock * ssk,struct mptcp_subflow_context * subflow,struct sk_buff * skb,struct mptcp_options_received * mp_opt)919 static bool check_fully_established(struct mptcp_sock *msk, struct sock *ssk,
920 				    struct mptcp_subflow_context *subflow,
921 				    struct sk_buff *skb,
922 				    struct mptcp_options_received *mp_opt)
923 {
924 	/* here we can process OoO, in-window pkts, only in-sequence 4th ack
925 	 * will make the subflow fully established
926 	 */
927 	if (likely(subflow->fully_established)) {
928 		/* on passive sockets, check for 3rd ack retransmission
929 		 * note that msk is always set by subflow_syn_recv_sock()
930 		 * for mp_join subflows
931 		 */
932 		if (TCP_SKB_CB(skb)->seq == subflow->ssn_offset + 1 &&
933 		    TCP_SKB_CB(skb)->end_seq == TCP_SKB_CB(skb)->seq &&
934 		    subflow->mp_join && (mp_opt->suboptions & OPTIONS_MPTCP_MPJ) &&
935 		    !subflow->request_join)
936 			tcp_send_ack(ssk);
937 		goto fully_established;
938 	}
939 
940 	/* we must process OoO packets before the first subflow is fully
941 	 * established. OoO packets are instead a protocol violation
942 	 * for MP_JOIN subflows as the peer must not send any data
943 	 * before receiving the forth ack - cfr. RFC 8684 section 3.2.
944 	 */
945 	if (TCP_SKB_CB(skb)->seq != subflow->ssn_offset + 1) {
946 		if (subflow->mp_join)
947 			goto reset;
948 		return subflow->mp_capable;
949 	}
950 
951 	if (((mp_opt->suboptions & OPTION_MPTCP_DSS) && mp_opt->use_ack) ||
952 	    ((mp_opt->suboptions & OPTION_MPTCP_ADD_ADDR) && !mp_opt->echo)) {
953 		/* subflows are fully established as soon as we get any
954 		 * additional ack, including ADD_ADDR.
955 		 */
956 		subflow->fully_established = 1;
957 		WRITE_ONCE(msk->fully_established, true);
958 		goto fully_established;
959 	}
960 
961 	/* If the first established packet does not contain MP_CAPABLE + data
962 	 * then fallback to TCP. Fallback scenarios requires a reset for
963 	 * MP_JOIN subflows.
964 	 */
965 	if (!(mp_opt->suboptions & OPTIONS_MPTCP_MPC)) {
966 		if (subflow->mp_join)
967 			goto reset;
968 		subflow->mp_capable = 0;
969 		pr_fallback(msk);
970 		mptcp_do_fallback(ssk);
971 		return false;
972 	}
973 
974 	if (mp_opt->deny_join_id0)
975 		WRITE_ONCE(msk->pm.remote_deny_join_id0, true);
976 
977 	if (unlikely(!READ_ONCE(msk->pm.server_side)))
978 		pr_warn_once("bogus mpc option on established client sk");
979 	mptcp_subflow_fully_established(subflow, mp_opt);
980 
981 fully_established:
982 	/* if the subflow is not already linked into the conn_list, we can't
983 	 * notify the PM: this subflow is still on the listener queue
984 	 * and the PM possibly acquiring the subflow lock could race with
985 	 * the listener close
986 	 */
987 	if (likely(subflow->pm_notified) || list_empty(&subflow->node))
988 		return true;
989 
990 	subflow->pm_notified = 1;
991 	if (subflow->mp_join) {
992 		clear_3rdack_retransmission(ssk);
993 		mptcp_pm_subflow_established(msk);
994 	} else {
995 		mptcp_pm_fully_established(msk, ssk, GFP_ATOMIC);
996 	}
997 	return true;
998 
999 reset:
1000 	mptcp_subflow_reset(ssk);
1001 	return false;
1002 }
1003 
__mptcp_expand_seq(u64 old_seq,u64 cur_seq)1004 u64 __mptcp_expand_seq(u64 old_seq, u64 cur_seq)
1005 {
1006 	u32 old_seq32, cur_seq32;
1007 
1008 	old_seq32 = (u32)old_seq;
1009 	cur_seq32 = (u32)cur_seq;
1010 	cur_seq = (old_seq & GENMASK_ULL(63, 32)) + cur_seq32;
1011 	if (unlikely(cur_seq32 < old_seq32 && before(old_seq32, cur_seq32)))
1012 		return cur_seq + (1LL << 32);
1013 
1014 	/* reverse wrap could happen, too */
1015 	if (unlikely(cur_seq32 > old_seq32 && after(old_seq32, cur_seq32)))
1016 		return cur_seq - (1LL << 32);
1017 	return cur_seq;
1018 }
1019 
ack_update_msk(struct mptcp_sock * msk,struct sock * ssk,struct mptcp_options_received * mp_opt)1020 static void ack_update_msk(struct mptcp_sock *msk,
1021 			   struct sock *ssk,
1022 			   struct mptcp_options_received *mp_opt)
1023 {
1024 	u64 new_wnd_end, new_snd_una, snd_nxt = READ_ONCE(msk->snd_nxt);
1025 	struct sock *sk = (struct sock *)msk;
1026 	u64 old_snd_una;
1027 
1028 	mptcp_data_lock(sk);
1029 
1030 	/* avoid ack expansion on update conflict, to reduce the risk of
1031 	 * wrongly expanding to a future ack sequence number, which is way
1032 	 * more dangerous than missing an ack
1033 	 */
1034 	old_snd_una = msk->snd_una;
1035 	new_snd_una = mptcp_expand_seq(old_snd_una, mp_opt->data_ack, mp_opt->ack64);
1036 
1037 	/* ACK for data not even sent yet? Ignore.*/
1038 	if (unlikely(after64(new_snd_una, snd_nxt)))
1039 		new_snd_una = old_snd_una;
1040 
1041 	new_wnd_end = new_snd_una + tcp_sk(ssk)->snd_wnd;
1042 
1043 	if (after64(new_wnd_end, msk->wnd_end))
1044 		msk->wnd_end = new_wnd_end;
1045 
1046 	/* this assumes mptcp_incoming_options() is invoked after tcp_ack() */
1047 	if (after64(msk->wnd_end, READ_ONCE(msk->snd_nxt)))
1048 		__mptcp_check_push(sk, ssk);
1049 
1050 	if (after64(new_snd_una, old_snd_una)) {
1051 		msk->snd_una = new_snd_una;
1052 		__mptcp_data_acked(sk);
1053 	}
1054 	mptcp_data_unlock(sk);
1055 
1056 	trace_ack_update_msk(mp_opt->data_ack,
1057 			     old_snd_una, new_snd_una,
1058 			     new_wnd_end, msk->wnd_end);
1059 }
1060 
mptcp_update_rcv_data_fin(struct mptcp_sock * msk,u64 data_fin_seq,bool use_64bit)1061 bool mptcp_update_rcv_data_fin(struct mptcp_sock *msk, u64 data_fin_seq, bool use_64bit)
1062 {
1063 	/* Skip if DATA_FIN was already received.
1064 	 * If updating simultaneously with the recvmsg loop, values
1065 	 * should match. If they mismatch, the peer is misbehaving and
1066 	 * we will prefer the most recent information.
1067 	 */
1068 	if (READ_ONCE(msk->rcv_data_fin))
1069 		return false;
1070 
1071 	WRITE_ONCE(msk->rcv_data_fin_seq,
1072 		   mptcp_expand_seq(READ_ONCE(msk->ack_seq), data_fin_seq, use_64bit));
1073 	WRITE_ONCE(msk->rcv_data_fin, 1);
1074 
1075 	return true;
1076 }
1077 
add_addr_hmac_valid(struct mptcp_sock * msk,struct mptcp_options_received * mp_opt)1078 static bool add_addr_hmac_valid(struct mptcp_sock *msk,
1079 				struct mptcp_options_received *mp_opt)
1080 {
1081 	u64 hmac = 0;
1082 
1083 	if (mp_opt->echo)
1084 		return true;
1085 
1086 	hmac = add_addr_generate_hmac(msk->remote_key,
1087 				      msk->local_key,
1088 				      &mp_opt->addr);
1089 
1090 	pr_debug("msk=%p, ahmac=%llu, mp_opt->ahmac=%llu\n",
1091 		 msk, hmac, mp_opt->ahmac);
1092 
1093 	return hmac == mp_opt->ahmac;
1094 }
1095 
1096 /* Return false if a subflow has been reset, else return true */
mptcp_incoming_options(struct sock * sk,struct sk_buff * skb)1097 bool mptcp_incoming_options(struct sock *sk, struct sk_buff *skb)
1098 {
1099 	struct mptcp_subflow_context *subflow = mptcp_subflow_ctx(sk);
1100 	struct mptcp_sock *msk = mptcp_sk(subflow->conn);
1101 	struct mptcp_options_received mp_opt;
1102 	struct mptcp_ext *mpext;
1103 
1104 	if (__mptcp_check_fallback(msk)) {
1105 		/* Keep it simple and unconditionally trigger send data cleanup and
1106 		 * pending queue spooling. We will need to acquire the data lock
1107 		 * for more accurate checks, and once the lock is acquired, such
1108 		 * helpers are cheap.
1109 		 */
1110 		mptcp_data_lock(subflow->conn);
1111 		if (sk_stream_memory_free(sk))
1112 			__mptcp_check_push(subflow->conn, sk);
1113 		__mptcp_data_acked(subflow->conn);
1114 		mptcp_data_unlock(subflow->conn);
1115 		return true;
1116 	}
1117 
1118 	mptcp_get_options(skb, &mp_opt);
1119 
1120 	/* The subflow can be in close state only if check_fully_established()
1121 	 * just sent a reset. If so, tell the caller to ignore the current packet.
1122 	 */
1123 	if (!check_fully_established(msk, sk, subflow, skb, &mp_opt))
1124 		return sk->sk_state != TCP_CLOSE;
1125 
1126 	if (unlikely(mp_opt.suboptions != OPTION_MPTCP_DSS)) {
1127 		if ((mp_opt.suboptions & OPTION_MPTCP_FASTCLOSE) &&
1128 		    msk->local_key == mp_opt.rcvr_key) {
1129 			WRITE_ONCE(msk->rcv_fastclose, true);
1130 			mptcp_schedule_work((struct sock *)msk);
1131 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFASTCLOSERX);
1132 		}
1133 
1134 		if ((mp_opt.suboptions & OPTION_MPTCP_ADD_ADDR) &&
1135 		    add_addr_hmac_valid(msk, &mp_opt)) {
1136 			if (!mp_opt.echo) {
1137 				mptcp_pm_add_addr_received(sk, &mp_opt.addr);
1138 				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ADDADDR);
1139 			} else {
1140 				mptcp_pm_add_addr_echoed(msk, &mp_opt.addr);
1141 				mptcp_pm_del_add_timer(msk, &mp_opt.addr, true);
1142 				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_ECHOADD);
1143 			}
1144 
1145 			if (mp_opt.addr.port)
1146 				MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_PORTADD);
1147 		}
1148 
1149 		if (mp_opt.suboptions & OPTION_MPTCP_RM_ADDR)
1150 			mptcp_pm_rm_addr_received(msk, &mp_opt.rm_list);
1151 
1152 		if (mp_opt.suboptions & OPTION_MPTCP_PRIO) {
1153 			mptcp_pm_mp_prio_received(sk, mp_opt.backup);
1154 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPPRIORX);
1155 		}
1156 
1157 		if (mp_opt.suboptions & OPTION_MPTCP_FAIL) {
1158 			mptcp_pm_mp_fail_received(sk, mp_opt.fail_seq);
1159 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPFAILRX);
1160 		}
1161 
1162 		if (mp_opt.suboptions & OPTION_MPTCP_RST) {
1163 			subflow->reset_seen = 1;
1164 			subflow->reset_reason = mp_opt.reset_reason;
1165 			subflow->reset_transient = mp_opt.reset_transient;
1166 			MPTCP_INC_STATS(sock_net(sk), MPTCP_MIB_MPRSTRX);
1167 		}
1168 
1169 		if (!(mp_opt.suboptions & OPTION_MPTCP_DSS))
1170 			return true;
1171 	}
1172 
1173 	/* we can't wait for recvmsg() to update the ack_seq, otherwise
1174 	 * monodirectional flows will stuck
1175 	 */
1176 	if (mp_opt.use_ack)
1177 		ack_update_msk(msk, sk, &mp_opt);
1178 
1179 	/* Zero-data-length packets are dropped by the caller and not
1180 	 * propagated to the MPTCP layer, so the skb extension does not
1181 	 * need to be allocated or populated. DATA_FIN information, if
1182 	 * present, needs to be updated here before the skb is freed.
1183 	 */
1184 	if (TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
1185 		if (mp_opt.data_fin && mp_opt.data_len == 1 &&
1186 		    mptcp_update_rcv_data_fin(msk, mp_opt.data_seq, mp_opt.dsn64) &&
1187 		    schedule_work(&msk->work))
1188 			sock_hold(subflow->conn);
1189 
1190 		return true;
1191 	}
1192 
1193 	mpext = skb_ext_add(skb, SKB_EXT_MPTCP);
1194 	if (!mpext)
1195 		return true;
1196 
1197 	memset(mpext, 0, sizeof(*mpext));
1198 
1199 	if (likely(mp_opt.use_map)) {
1200 		if (mp_opt.mpc_map) {
1201 			/* this is an MP_CAPABLE carrying MPTCP data
1202 			 * we know this map the first chunk of data
1203 			 */
1204 			mptcp_crypto_key_sha(subflow->remote_key, NULL,
1205 					     &mpext->data_seq);
1206 			mpext->data_seq++;
1207 			mpext->subflow_seq = 1;
1208 			mpext->dsn64 = 1;
1209 			mpext->mpc_map = 1;
1210 			mpext->data_fin = 0;
1211 		} else {
1212 			mpext->data_seq = mp_opt.data_seq;
1213 			mpext->subflow_seq = mp_opt.subflow_seq;
1214 			mpext->dsn64 = mp_opt.dsn64;
1215 			mpext->data_fin = mp_opt.data_fin;
1216 		}
1217 		mpext->data_len = mp_opt.data_len;
1218 		mpext->use_map = 1;
1219 		mpext->csum_reqd = !!(mp_opt.suboptions & OPTION_MPTCP_CSUMREQD);
1220 
1221 		if (mpext->csum_reqd)
1222 			mpext->csum = mp_opt.csum;
1223 	}
1224 
1225 	return true;
1226 }
1227 
mptcp_set_rwin(struct tcp_sock * tp,struct tcphdr * th)1228 static void mptcp_set_rwin(struct tcp_sock *tp, struct tcphdr *th)
1229 {
1230 	const struct sock *ssk = (const struct sock *)tp;
1231 	struct mptcp_subflow_context *subflow;
1232 	u64 ack_seq, rcv_wnd_old, rcv_wnd_new;
1233 	struct mptcp_sock *msk;
1234 	u32 new_win;
1235 	u64 win;
1236 
1237 	subflow = mptcp_subflow_ctx(ssk);
1238 	msk = mptcp_sk(subflow->conn);
1239 
1240 	ack_seq = READ_ONCE(msk->ack_seq);
1241 	rcv_wnd_new = ack_seq + tp->rcv_wnd;
1242 
1243 	rcv_wnd_old = atomic64_read(&msk->rcv_wnd_sent);
1244 	if (after64(rcv_wnd_new, rcv_wnd_old)) {
1245 		u64 rcv_wnd;
1246 
1247 		for (;;) {
1248 			rcv_wnd = atomic64_cmpxchg(&msk->rcv_wnd_sent, rcv_wnd_old, rcv_wnd_new);
1249 
1250 			if (rcv_wnd == rcv_wnd_old)
1251 				break;
1252 			if (before64(rcv_wnd_new, rcv_wnd)) {
1253 				MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICTUPDATE);
1254 				goto raise_win;
1255 			}
1256 			MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDCONFLICT);
1257 			rcv_wnd_old = rcv_wnd;
1258 		}
1259 		return;
1260 	}
1261 
1262 	if (rcv_wnd_new != rcv_wnd_old) {
1263 raise_win:
1264 		win = rcv_wnd_old - ack_seq;
1265 		tp->rcv_wnd = min_t(u64, win, U32_MAX);
1266 		new_win = tp->rcv_wnd;
1267 
1268 		/* Make sure we do not exceed the maximum possible
1269 		 * scaled window.
1270 		 */
1271 		if (unlikely(th->syn))
1272 			new_win = min(new_win, 65535U) << tp->rx_opt.rcv_wscale;
1273 		if (!tp->rx_opt.rcv_wscale &&
1274 		    READ_ONCE(sock_net(ssk)->ipv4.sysctl_tcp_workaround_signed_windows))
1275 			new_win = min(new_win, MAX_TCP_WINDOW);
1276 		else
1277 			new_win = min(new_win, (65535U << tp->rx_opt.rcv_wscale));
1278 
1279 		/* RFC1323 scaling applied */
1280 		new_win >>= tp->rx_opt.rcv_wscale;
1281 		th->window = htons(new_win);
1282 		MPTCP_INC_STATS(sock_net(ssk), MPTCP_MIB_RCVWNDSHARED);
1283 	}
1284 }
1285 
__mptcp_make_csum(u64 data_seq,u32 subflow_seq,u16 data_len,__wsum sum)1286 __sum16 __mptcp_make_csum(u64 data_seq, u32 subflow_seq, u16 data_len, __wsum sum)
1287 {
1288 	struct csum_pseudo_header header;
1289 	__wsum csum;
1290 
1291 	/* cfr RFC 8684 3.3.1.:
1292 	 * the data sequence number used in the pseudo-header is
1293 	 * always the 64-bit value, irrespective of what length is used in the
1294 	 * DSS option itself.
1295 	 */
1296 	header.data_seq = cpu_to_be64(data_seq);
1297 	header.subflow_seq = htonl(subflow_seq);
1298 	header.data_len = htons(data_len);
1299 	header.csum = 0;
1300 
1301 	csum = csum_partial(&header, sizeof(header), sum);
1302 	return csum_fold(csum);
1303 }
1304 
mptcp_make_csum(const struct mptcp_ext * mpext)1305 static __sum16 mptcp_make_csum(const struct mptcp_ext *mpext)
1306 {
1307 	return __mptcp_make_csum(mpext->data_seq, mpext->subflow_seq, mpext->data_len,
1308 				 ~csum_unfold(mpext->csum));
1309 }
1310 
put_len_csum(u16 len,__sum16 csum,void * data)1311 static void put_len_csum(u16 len, __sum16 csum, void *data)
1312 {
1313 	__sum16 *sumptr = data + 2;
1314 	__be16 *ptr = data;
1315 
1316 	put_unaligned_be16(len, ptr);
1317 
1318 	put_unaligned(csum, sumptr);
1319 }
1320 
mptcp_write_options(struct tcphdr * th,__be32 * ptr,struct tcp_sock * tp,struct mptcp_out_options * opts)1321 void mptcp_write_options(struct tcphdr *th, __be32 *ptr, struct tcp_sock *tp,
1322 			 struct mptcp_out_options *opts)
1323 {
1324 	const struct sock *ssk = (const struct sock *)tp;
1325 	struct mptcp_subflow_context *subflow;
1326 
1327 	/* Which options can be used together?
1328 	 *
1329 	 * X: mutually exclusive
1330 	 * O: often used together
1331 	 * C: can be used together in some cases
1332 	 * P: could be used together but we prefer not to (optimisations)
1333 	 *
1334 	 *  Opt: | MPC  | MPJ  | DSS  | ADD  |  RM  | PRIO | FAIL |  FC  |
1335 	 * ------|------|------|------|------|------|------|------|------|
1336 	 *  MPC  |------|------|------|------|------|------|------|------|
1337 	 *  MPJ  |  X   |------|------|------|------|------|------|------|
1338 	 *  DSS  |  X   |  X   |------|------|------|------|------|------|
1339 	 *  ADD  |  X   |  X   |  P   |------|------|------|------|------|
1340 	 *  RM   |  C   |  C   |  C   |  P   |------|------|------|------|
1341 	 *  PRIO |  X   |  C   |  C   |  C   |  C   |------|------|------|
1342 	 *  FAIL |  X   |  X   |  C   |  X   |  X   |  X   |------|------|
1343 	 *  FC   |  X   |  X   |  X   |  X   |  X   |  X   |  X   |------|
1344 	 *  RST  |  X   |  X   |  X   |  X   |  X   |  X   |  O   |  O   |
1345 	 * ------|------|------|------|------|------|------|------|------|
1346 	 *
1347 	 * The same applies in mptcp_established_options() function.
1348 	 */
1349 	if (likely(OPTION_MPTCP_DSS & opts->suboptions)) {
1350 		struct mptcp_ext *mpext = &opts->ext_copy;
1351 		u8 len = TCPOLEN_MPTCP_DSS_BASE;
1352 		u8 flags = 0;
1353 
1354 		if (mpext->use_ack) {
1355 			flags = MPTCP_DSS_HAS_ACK;
1356 			if (mpext->ack64) {
1357 				len += TCPOLEN_MPTCP_DSS_ACK64;
1358 				flags |= MPTCP_DSS_ACK64;
1359 			} else {
1360 				len += TCPOLEN_MPTCP_DSS_ACK32;
1361 			}
1362 		}
1363 
1364 		if (mpext->use_map) {
1365 			len += TCPOLEN_MPTCP_DSS_MAP64;
1366 
1367 			/* Use only 64-bit mapping flags for now, add
1368 			 * support for optional 32-bit mappings later.
1369 			 */
1370 			flags |= MPTCP_DSS_HAS_MAP | MPTCP_DSS_DSN64;
1371 			if (mpext->data_fin)
1372 				flags |= MPTCP_DSS_DATA_FIN;
1373 
1374 			if (opts->csum_reqd)
1375 				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
1376 		}
1377 
1378 		*ptr++ = mptcp_option(MPTCPOPT_DSS, len, 0, flags);
1379 
1380 		if (mpext->use_ack) {
1381 			if (mpext->ack64) {
1382 				put_unaligned_be64(mpext->data_ack, ptr);
1383 				ptr += 2;
1384 			} else {
1385 				put_unaligned_be32(mpext->data_ack32, ptr);
1386 				ptr += 1;
1387 			}
1388 		}
1389 
1390 		if (mpext->use_map) {
1391 			put_unaligned_be64(mpext->data_seq, ptr);
1392 			ptr += 2;
1393 			put_unaligned_be32(mpext->subflow_seq, ptr);
1394 			ptr += 1;
1395 			if (opts->csum_reqd) {
1396 				/* data_len == 0 is reserved for the infinite mapping,
1397 				 * the checksum will also be set to 0.
1398 				 */
1399 				put_len_csum(mpext->data_len,
1400 					     (mpext->data_len ? mptcp_make_csum(mpext) : 0),
1401 					     ptr);
1402 			} else {
1403 				put_unaligned_be32(mpext->data_len << 16 |
1404 						   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
1405 			}
1406 			ptr += 1;
1407 		}
1408 
1409 		/* We might need to add MP_FAIL options in rare cases */
1410 		if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions))
1411 			goto mp_fail;
1412 	} else if (OPTIONS_MPTCP_MPC & opts->suboptions) {
1413 		u8 len, flag = MPTCP_CAP_HMAC_SHA256;
1414 
1415 		if (OPTION_MPTCP_MPC_SYN & opts->suboptions) {
1416 			len = TCPOLEN_MPTCP_MPC_SYN;
1417 		} else if (OPTION_MPTCP_MPC_SYNACK & opts->suboptions) {
1418 			len = TCPOLEN_MPTCP_MPC_SYNACK;
1419 		} else if (opts->data_len) {
1420 			len = TCPOLEN_MPTCP_MPC_ACK_DATA;
1421 			if (opts->csum_reqd)
1422 				len += TCPOLEN_MPTCP_DSS_CHECKSUM;
1423 		} else {
1424 			len = TCPOLEN_MPTCP_MPC_ACK;
1425 		}
1426 
1427 		if (opts->csum_reqd)
1428 			flag |= MPTCP_CAP_CHECKSUM_REQD;
1429 
1430 		if (!opts->allow_join_id0)
1431 			flag |= MPTCP_CAP_DENY_JOIN_ID0;
1432 
1433 		*ptr++ = mptcp_option(MPTCPOPT_MP_CAPABLE, len,
1434 				      MPTCP_SUPPORTED_VERSION,
1435 				      flag);
1436 
1437 		if (!((OPTION_MPTCP_MPC_SYNACK | OPTION_MPTCP_MPC_ACK) &
1438 		    opts->suboptions))
1439 			goto mp_capable_done;
1440 
1441 		put_unaligned_be64(opts->sndr_key, ptr);
1442 		ptr += 2;
1443 		if (!((OPTION_MPTCP_MPC_ACK) & opts->suboptions))
1444 			goto mp_capable_done;
1445 
1446 		put_unaligned_be64(opts->rcvr_key, ptr);
1447 		ptr += 2;
1448 		if (!opts->data_len)
1449 			goto mp_capable_done;
1450 
1451 		if (opts->csum_reqd) {
1452 			put_len_csum(opts->data_len,
1453 				     __mptcp_make_csum(opts->data_seq,
1454 						       opts->subflow_seq,
1455 						       opts->data_len,
1456 						       ~csum_unfold(opts->csum)),
1457 				     ptr);
1458 		} else {
1459 			put_unaligned_be32(opts->data_len << 16 |
1460 					   TCPOPT_NOP << 8 | TCPOPT_NOP, ptr);
1461 		}
1462 		ptr += 1;
1463 
1464 		/* MPC is additionally mutually exclusive with MP_PRIO */
1465 		goto mp_capable_done;
1466 	} else if (OPTIONS_MPTCP_MPJ & opts->suboptions) {
1467 		if (OPTION_MPTCP_MPJ_SYN & opts->suboptions) {
1468 			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1469 					      TCPOLEN_MPTCP_MPJ_SYN,
1470 					      opts->backup, opts->join_id);
1471 			put_unaligned_be32(opts->token, ptr);
1472 			ptr += 1;
1473 			put_unaligned_be32(opts->nonce, ptr);
1474 			ptr += 1;
1475 		} else if (OPTION_MPTCP_MPJ_SYNACK & opts->suboptions) {
1476 			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1477 					      TCPOLEN_MPTCP_MPJ_SYNACK,
1478 					      opts->backup, opts->join_id);
1479 			put_unaligned_be64(opts->thmac, ptr);
1480 			ptr += 2;
1481 			put_unaligned_be32(opts->nonce, ptr);
1482 			ptr += 1;
1483 		} else {
1484 			*ptr++ = mptcp_option(MPTCPOPT_MP_JOIN,
1485 					      TCPOLEN_MPTCP_MPJ_ACK, 0, 0);
1486 			memcpy(ptr, opts->hmac, MPTCPOPT_HMAC_LEN);
1487 			ptr += 5;
1488 		}
1489 	} else if (OPTION_MPTCP_ADD_ADDR & opts->suboptions) {
1490 		u8 len = TCPOLEN_MPTCP_ADD_ADDR_BASE;
1491 		u8 echo = MPTCP_ADDR_ECHO;
1492 
1493 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1494 		if (opts->addr.family == AF_INET6)
1495 			len = TCPOLEN_MPTCP_ADD_ADDR6_BASE;
1496 #endif
1497 
1498 		if (opts->addr.port)
1499 			len += TCPOLEN_MPTCP_PORT_LEN;
1500 
1501 		if (opts->ahmac) {
1502 			len += sizeof(opts->ahmac);
1503 			echo = 0;
1504 		}
1505 
1506 		*ptr++ = mptcp_option(MPTCPOPT_ADD_ADDR,
1507 				      len, echo, opts->addr.id);
1508 		if (opts->addr.family == AF_INET) {
1509 			memcpy((u8 *)ptr, (u8 *)&opts->addr.addr.s_addr, 4);
1510 			ptr += 1;
1511 		}
1512 #if IS_ENABLED(CONFIG_MPTCP_IPV6)
1513 		else if (opts->addr.family == AF_INET6) {
1514 			memcpy((u8 *)ptr, opts->addr.addr6.s6_addr, 16);
1515 			ptr += 4;
1516 		}
1517 #endif
1518 
1519 		if (!opts->addr.port) {
1520 			if (opts->ahmac) {
1521 				put_unaligned_be64(opts->ahmac, ptr);
1522 				ptr += 2;
1523 			}
1524 		} else {
1525 			u16 port = ntohs(opts->addr.port);
1526 
1527 			if (opts->ahmac) {
1528 				u8 *bptr = (u8 *)ptr;
1529 
1530 				put_unaligned_be16(port, bptr);
1531 				bptr += 2;
1532 				put_unaligned_be64(opts->ahmac, bptr);
1533 				bptr += 8;
1534 				put_unaligned_be16(TCPOPT_NOP << 8 |
1535 						   TCPOPT_NOP, bptr);
1536 
1537 				ptr += 3;
1538 			} else {
1539 				put_unaligned_be32(port << 16 |
1540 						   TCPOPT_NOP << 8 |
1541 						   TCPOPT_NOP, ptr);
1542 				ptr += 1;
1543 			}
1544 		}
1545 	} else if (unlikely(OPTION_MPTCP_FASTCLOSE & opts->suboptions)) {
1546 		/* FASTCLOSE is mutually exclusive with others except RST */
1547 		*ptr++ = mptcp_option(MPTCPOPT_MP_FASTCLOSE,
1548 				      TCPOLEN_MPTCP_FASTCLOSE,
1549 				      0, 0);
1550 		put_unaligned_be64(opts->rcvr_key, ptr);
1551 		ptr += 2;
1552 
1553 		if (OPTION_MPTCP_RST & opts->suboptions)
1554 			goto mp_rst;
1555 		return;
1556 	} else if (unlikely(OPTION_MPTCP_FAIL & opts->suboptions)) {
1557 mp_fail:
1558 		/* MP_FAIL is mutually exclusive with others except RST */
1559 		subflow = mptcp_subflow_ctx(ssk);
1560 		subflow->send_mp_fail = 0;
1561 
1562 		*ptr++ = mptcp_option(MPTCPOPT_MP_FAIL,
1563 				      TCPOLEN_MPTCP_FAIL,
1564 				      0, 0);
1565 		put_unaligned_be64(opts->fail_seq, ptr);
1566 		ptr += 2;
1567 
1568 		if (OPTION_MPTCP_RST & opts->suboptions)
1569 			goto mp_rst;
1570 		return;
1571 	} else if (unlikely(OPTION_MPTCP_RST & opts->suboptions)) {
1572 mp_rst:
1573 		*ptr++ = mptcp_option(MPTCPOPT_RST,
1574 				      TCPOLEN_MPTCP_RST,
1575 				      opts->reset_transient,
1576 				      opts->reset_reason);
1577 		return;
1578 	}
1579 
1580 	if (OPTION_MPTCP_PRIO & opts->suboptions) {
1581 		subflow = mptcp_subflow_ctx(ssk);
1582 		subflow->send_mp_prio = 0;
1583 
1584 		*ptr++ = mptcp_option(MPTCPOPT_MP_PRIO,
1585 				      TCPOLEN_MPTCP_PRIO,
1586 				      opts->backup, TCPOPT_NOP);
1587 
1588 		MPTCP_INC_STATS(sock_net((const struct sock *)tp),
1589 				MPTCP_MIB_MPPRIOTX);
1590 	}
1591 
1592 mp_capable_done:
1593 	if (OPTION_MPTCP_RM_ADDR & opts->suboptions) {
1594 		u8 i = 1;
1595 
1596 		*ptr++ = mptcp_option(MPTCPOPT_RM_ADDR,
1597 				      TCPOLEN_MPTCP_RM_ADDR_BASE + opts->rm_list.nr,
1598 				      0, opts->rm_list.ids[0]);
1599 
1600 		while (i < opts->rm_list.nr) {
1601 			u8 id1, id2, id3, id4;
1602 
1603 			id1 = opts->rm_list.ids[i];
1604 			id2 = i + 1 < opts->rm_list.nr ? opts->rm_list.ids[i + 1] : TCPOPT_NOP;
1605 			id3 = i + 2 < opts->rm_list.nr ? opts->rm_list.ids[i + 2] : TCPOPT_NOP;
1606 			id4 = i + 3 < opts->rm_list.nr ? opts->rm_list.ids[i + 3] : TCPOPT_NOP;
1607 			put_unaligned_be32(id1 << 24 | id2 << 16 | id3 << 8 | id4, ptr);
1608 			ptr += 1;
1609 			i += 4;
1610 		}
1611 	}
1612 
1613 	if (tp)
1614 		mptcp_set_rwin(tp, th);
1615 }
1616 
mptcp_get_reset_option(const struct sk_buff * skb)1617 __be32 mptcp_get_reset_option(const struct sk_buff *skb)
1618 {
1619 	const struct mptcp_ext *ext = mptcp_get_ext(skb);
1620 	u8 flags, reason;
1621 
1622 	if (ext) {
1623 		flags = ext->reset_transient;
1624 		reason = ext->reset_reason;
1625 
1626 		return mptcp_option(MPTCPOPT_RST, TCPOLEN_MPTCP_RST,
1627 				    flags, reason);
1628 	}
1629 
1630 	return htonl(0u);
1631 }
1632 EXPORT_SYMBOL_GPL(mptcp_get_reset_option);
1633