1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *
9 * IPv4 specific functions
10 *
11 * code split from:
12 * linux/ipv4/tcp.c
13 * linux/ipv4/tcp_input.c
14 * linux/ipv4/tcp_output.c
15 *
16 * See tcp.c for author information
17 */
18
19 /*
20 * Changes:
21 * David S. Miller : New socket lookup architecture.
22 * This code is dedicated to John Dyson.
23 * David S. Miller : Change semantics of established hash,
24 * half is devoted to TIME_WAIT sockets
25 * and the rest go in the other half.
26 * Andi Kleen : Add support for syncookies and fixed
27 * some bugs: ip options weren't passed to
28 * the TCP layer, missed a check for an
29 * ACK bit.
30 * Andi Kleen : Implemented fast path mtu discovery.
31 * Fixed many serious bugs in the
32 * request_sock handling and moved
33 * most of it into the af independent code.
34 * Added tail drop and some other bugfixes.
35 * Added new listen semantics.
36 * Mike McLagan : Routing by source
37 * Juan Jose Ciarlante: ip_dynaddr bits
38 * Andi Kleen: various fixes.
39 * Vitaly E. Lavrov : Transparent proxy revived after year
40 * coma.
41 * Andi Kleen : Fix new listen.
42 * Andi Kleen : Fix accept error reporting.
43 * YOSHIFUJI Hideaki @USAGI and: Support IPV6_V6ONLY socket option, which
44 * Alexey Kuznetsov allow both IPv4 and IPv6 sockets to bind
45 * a single port at the same time.
46 */
47
48 #define pr_fmt(fmt) "TCP: " fmt
49
50 #include <linux/bottom_half.h>
51 #include <linux/types.h>
52 #include <linux/fcntl.h>
53 #include <linux/module.h>
54 #include <linux/random.h>
55 #include <linux/cache.h>
56 #include <linux/jhash.h>
57 #include <linux/init.h>
58 #include <linux/times.h>
59 #include <linux/slab.h>
60
61 #include <net/net_namespace.h>
62 #include <net/icmp.h>
63 #include <net/inet_hashtables.h>
64 #include <net/tcp.h>
65 #include <net/transp_v6.h>
66 #include <net/ipv6.h>
67 #include <net/inet_common.h>
68 #include <net/timewait_sock.h>
69 #include <net/xfrm.h>
70 #include <net/secure_seq.h>
71 #include <net/busy_poll.h>
72
73 #include <linux/inet.h>
74 #include <linux/ipv6.h>
75 #include <linux/stddef.h>
76 #include <linux/proc_fs.h>
77 #include <linux/seq_file.h>
78 #include <linux/inetdevice.h>
79 #include <linux/btf_ids.h>
80
81 #include <crypto/hash.h>
82 #include <linux/scatterlist.h>
83
84 #include <trace/events/tcp.h>
85
86 #ifdef CONFIG_TCP_MD5SIG
87 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
88 __be32 daddr, __be32 saddr, const struct tcphdr *th);
89 #endif
90
91 struct inet_hashinfo tcp_hashinfo;
92 EXPORT_SYMBOL(tcp_hashinfo);
93
94 static DEFINE_PER_CPU(struct sock *, ipv4_tcp_sk);
95
tcp_v4_init_seq(const struct sk_buff * skb)96 static u32 tcp_v4_init_seq(const struct sk_buff *skb)
97 {
98 return secure_tcp_seq(ip_hdr(skb)->daddr,
99 ip_hdr(skb)->saddr,
100 tcp_hdr(skb)->dest,
101 tcp_hdr(skb)->source);
102 }
103
tcp_v4_init_ts_off(const struct net * net,const struct sk_buff * skb)104 static u32 tcp_v4_init_ts_off(const struct net *net, const struct sk_buff *skb)
105 {
106 return secure_tcp_ts_off(net, ip_hdr(skb)->daddr, ip_hdr(skb)->saddr);
107 }
108
tcp_twsk_unique(struct sock * sk,struct sock * sktw,void * twp)109 int tcp_twsk_unique(struct sock *sk, struct sock *sktw, void *twp)
110 {
111 int reuse = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_tw_reuse);
112 const struct inet_timewait_sock *tw = inet_twsk(sktw);
113 const struct tcp_timewait_sock *tcptw = tcp_twsk(sktw);
114 struct tcp_sock *tp = tcp_sk(sk);
115
116 if (reuse == 2) {
117 /* Still does not detect *everything* that goes through
118 * lo, since we require a loopback src or dst address
119 * or direct binding to 'lo' interface.
120 */
121 bool loopback = false;
122 if (tw->tw_bound_dev_if == LOOPBACK_IFINDEX)
123 loopback = true;
124 #if IS_ENABLED(CONFIG_IPV6)
125 if (tw->tw_family == AF_INET6) {
126 if (ipv6_addr_loopback(&tw->tw_v6_daddr) ||
127 ipv6_addr_v4mapped_loopback(&tw->tw_v6_daddr) ||
128 ipv6_addr_loopback(&tw->tw_v6_rcv_saddr) ||
129 ipv6_addr_v4mapped_loopback(&tw->tw_v6_rcv_saddr))
130 loopback = true;
131 } else
132 #endif
133 {
134 if (ipv4_is_loopback(tw->tw_daddr) ||
135 ipv4_is_loopback(tw->tw_rcv_saddr))
136 loopback = true;
137 }
138 if (!loopback)
139 reuse = 0;
140 }
141
142 /* With PAWS, it is safe from the viewpoint
143 of data integrity. Even without PAWS it is safe provided sequence
144 spaces do not overlap i.e. at data rates <= 80Mbit/sec.
145
146 Actually, the idea is close to VJ's one, only timestamp cache is
147 held not per host, but per port pair and TW bucket is used as state
148 holder.
149
150 If TW bucket has been already destroyed we fall back to VJ's scheme
151 and use initial timestamp retrieved from peer table.
152 */
153 if (tcptw->tw_ts_recent_stamp &&
154 (!twp || (reuse && time_after32(ktime_get_seconds(),
155 tcptw->tw_ts_recent_stamp)))) {
156 /* In case of repair and re-using TIME-WAIT sockets we still
157 * want to be sure that it is safe as above but honor the
158 * sequence numbers and time stamps set as part of the repair
159 * process.
160 *
161 * Without this check re-using a TIME-WAIT socket with TCP
162 * repair would accumulate a -1 on the repair assigned
163 * sequence number. The first time it is reused the sequence
164 * is -1, the second time -2, etc. This fixes that issue
165 * without appearing to create any others.
166 */
167 if (likely(!tp->repair)) {
168 u32 seq = tcptw->tw_snd_nxt + 65535 + 2;
169
170 if (!seq)
171 seq = 1;
172 WRITE_ONCE(tp->write_seq, seq);
173 tp->rx_opt.ts_recent = tcptw->tw_ts_recent;
174 tp->rx_opt.ts_recent_stamp = tcptw->tw_ts_recent_stamp;
175 }
176 sock_hold(sktw);
177 return 1;
178 }
179
180 return 0;
181 }
182 EXPORT_SYMBOL_GPL(tcp_twsk_unique);
183
tcp_v4_pre_connect(struct sock * sk,struct sockaddr * uaddr,int addr_len)184 static int tcp_v4_pre_connect(struct sock *sk, struct sockaddr *uaddr,
185 int addr_len)
186 {
187 /* This check is replicated from tcp_v4_connect() and intended to
188 * prevent BPF program called below from accessing bytes that are out
189 * of the bound specified by user in addr_len.
190 */
191 if (addr_len < sizeof(struct sockaddr_in))
192 return -EINVAL;
193
194 sock_owned_by_me(sk);
195
196 return BPF_CGROUP_RUN_PROG_INET4_CONNECT(sk, uaddr);
197 }
198
199 /* This will initiate an outgoing connection. */
tcp_v4_connect(struct sock * sk,struct sockaddr * uaddr,int addr_len)200 int tcp_v4_connect(struct sock *sk, struct sockaddr *uaddr, int addr_len)
201 {
202 struct sockaddr_in *usin = (struct sockaddr_in *)uaddr;
203 struct inet_timewait_death_row *tcp_death_row;
204 struct inet_sock *inet = inet_sk(sk);
205 struct tcp_sock *tp = tcp_sk(sk);
206 struct ip_options_rcu *inet_opt;
207 struct net *net = sock_net(sk);
208 __be16 orig_sport, orig_dport;
209 __be32 daddr, nexthop;
210 struct flowi4 *fl4;
211 struct rtable *rt;
212 int err;
213
214 if (addr_len < sizeof(struct sockaddr_in))
215 return -EINVAL;
216
217 if (usin->sin_family != AF_INET)
218 return -EAFNOSUPPORT;
219
220 nexthop = daddr = usin->sin_addr.s_addr;
221 inet_opt = rcu_dereference_protected(inet->inet_opt,
222 lockdep_sock_is_held(sk));
223 if (inet_opt && inet_opt->opt.srr) {
224 if (!daddr)
225 return -EINVAL;
226 nexthop = inet_opt->opt.faddr;
227 }
228
229 orig_sport = inet->inet_sport;
230 orig_dport = usin->sin_port;
231 fl4 = &inet->cork.fl.u.ip4;
232 rt = ip_route_connect(fl4, nexthop, inet->inet_saddr,
233 sk->sk_bound_dev_if, IPPROTO_TCP, orig_sport,
234 orig_dport, sk);
235 if (IS_ERR(rt)) {
236 err = PTR_ERR(rt);
237 if (err == -ENETUNREACH)
238 IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
239 return err;
240 }
241
242 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) {
243 ip_rt_put(rt);
244 return -ENETUNREACH;
245 }
246
247 if (!inet_opt || !inet_opt->opt.srr)
248 daddr = fl4->daddr;
249
250 tcp_death_row = &sock_net(sk)->ipv4.tcp_death_row;
251
252 if (!inet->inet_saddr) {
253 err = inet_bhash2_update_saddr(sk, &fl4->saddr, AF_INET);
254 if (err) {
255 ip_rt_put(rt);
256 return err;
257 }
258 } else {
259 sk_rcv_saddr_set(sk, inet->inet_saddr);
260 }
261
262 if (tp->rx_opt.ts_recent_stamp && inet->inet_daddr != daddr) {
263 /* Reset inherited state */
264 tp->rx_opt.ts_recent = 0;
265 tp->rx_opt.ts_recent_stamp = 0;
266 if (likely(!tp->repair))
267 WRITE_ONCE(tp->write_seq, 0);
268 }
269
270 inet->inet_dport = usin->sin_port;
271 sk_daddr_set(sk, daddr);
272
273 inet_csk(sk)->icsk_ext_hdr_len = 0;
274 if (inet_opt)
275 inet_csk(sk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
276
277 tp->rx_opt.mss_clamp = TCP_MSS_DEFAULT;
278
279 /* Socket identity is still unknown (sport may be zero).
280 * However we set state to SYN-SENT and not releasing socket
281 * lock select source port, enter ourselves into the hash tables and
282 * complete initialization after this.
283 */
284 tcp_set_state(sk, TCP_SYN_SENT);
285 err = inet_hash_connect(tcp_death_row, sk);
286 if (err)
287 goto failure;
288
289 sk_set_txhash(sk);
290
291 rt = ip_route_newports(fl4, rt, orig_sport, orig_dport,
292 inet->inet_sport, inet->inet_dport, sk);
293 if (IS_ERR(rt)) {
294 err = PTR_ERR(rt);
295 rt = NULL;
296 goto failure;
297 }
298 /* OK, now commit destination to socket. */
299 sk->sk_gso_type = SKB_GSO_TCPV4;
300 sk_setup_caps(sk, &rt->dst);
301 rt = NULL;
302
303 if (likely(!tp->repair)) {
304 if (!tp->write_seq)
305 WRITE_ONCE(tp->write_seq,
306 secure_tcp_seq(inet->inet_saddr,
307 inet->inet_daddr,
308 inet->inet_sport,
309 usin->sin_port));
310 tp->tsoffset = secure_tcp_ts_off(net, inet->inet_saddr,
311 inet->inet_daddr);
312 }
313
314 inet->inet_id = get_random_u16();
315
316 if (tcp_fastopen_defer_connect(sk, &err))
317 return err;
318 if (err)
319 goto failure;
320
321 err = tcp_connect(sk);
322
323 if (err)
324 goto failure;
325
326 return 0;
327
328 failure:
329 /*
330 * This unhashes the socket and releases the local port,
331 * if necessary.
332 */
333 tcp_set_state(sk, TCP_CLOSE);
334 inet_bhash2_reset_saddr(sk);
335 ip_rt_put(rt);
336 sk->sk_route_caps = 0;
337 inet->inet_dport = 0;
338 return err;
339 }
340 EXPORT_SYMBOL(tcp_v4_connect);
341
342 /*
343 * This routine reacts to ICMP_FRAG_NEEDED mtu indications as defined in RFC1191.
344 * It can be called through tcp_release_cb() if socket was owned by user
345 * at the time tcp_v4_err() was called to handle ICMP message.
346 */
tcp_v4_mtu_reduced(struct sock * sk)347 void tcp_v4_mtu_reduced(struct sock *sk)
348 {
349 struct inet_sock *inet = inet_sk(sk);
350 struct dst_entry *dst;
351 u32 mtu;
352
353 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
354 return;
355 mtu = READ_ONCE(tcp_sk(sk)->mtu_info);
356 dst = inet_csk_update_pmtu(sk, mtu);
357 if (!dst)
358 return;
359
360 /* Something is about to be wrong... Remember soft error
361 * for the case, if this connection will not able to recover.
362 */
363 if (mtu < dst_mtu(dst) && ip_dont_fragment(sk, dst))
364 sk->sk_err_soft = EMSGSIZE;
365
366 mtu = dst_mtu(dst);
367
368 if (inet->pmtudisc != IP_PMTUDISC_DONT &&
369 ip_sk_accept_pmtu(sk) &&
370 inet_csk(sk)->icsk_pmtu_cookie > mtu) {
371 tcp_sync_mss(sk, mtu);
372
373 /* Resend the TCP packet because it's
374 * clear that the old packet has been
375 * dropped. This is the new "fast" path mtu
376 * discovery.
377 */
378 tcp_simple_retransmit(sk);
379 } /* else let the usual retransmit timer handle it */
380 }
381 EXPORT_SYMBOL(tcp_v4_mtu_reduced);
382
do_redirect(struct sk_buff * skb,struct sock * sk)383 static void do_redirect(struct sk_buff *skb, struct sock *sk)
384 {
385 struct dst_entry *dst = __sk_dst_check(sk, 0);
386
387 if (dst)
388 dst->ops->redirect(dst, sk, skb);
389 }
390
391
392 /* handle ICMP messages on TCP_NEW_SYN_RECV request sockets */
tcp_req_err(struct sock * sk,u32 seq,bool abort)393 void tcp_req_err(struct sock *sk, u32 seq, bool abort)
394 {
395 struct request_sock *req = inet_reqsk(sk);
396 struct net *net = sock_net(sk);
397
398 /* ICMPs are not backlogged, hence we cannot get
399 * an established socket here.
400 */
401 if (seq != tcp_rsk(req)->snt_isn) {
402 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
403 } else if (abort) {
404 /*
405 * Still in SYN_RECV, just remove it silently.
406 * There is no good way to pass the error to the newly
407 * created socket, and POSIX does not want network
408 * errors returned from accept().
409 */
410 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
411 tcp_listendrop(req->rsk_listener);
412 }
413 reqsk_put(req);
414 }
415 EXPORT_SYMBOL(tcp_req_err);
416
417 /* TCP-LD (RFC 6069) logic */
tcp_ld_RTO_revert(struct sock * sk,u32 seq)418 void tcp_ld_RTO_revert(struct sock *sk, u32 seq)
419 {
420 struct inet_connection_sock *icsk = inet_csk(sk);
421 struct tcp_sock *tp = tcp_sk(sk);
422 struct sk_buff *skb;
423 s32 remaining;
424 u32 delta_us;
425
426 if (sock_owned_by_user(sk))
427 return;
428
429 if (seq != tp->snd_una || !icsk->icsk_retransmits ||
430 !icsk->icsk_backoff)
431 return;
432
433 skb = tcp_rtx_queue_head(sk);
434 if (WARN_ON_ONCE(!skb))
435 return;
436
437 icsk->icsk_backoff--;
438 icsk->icsk_rto = tp->srtt_us ? __tcp_set_rto(tp) : TCP_TIMEOUT_INIT;
439 icsk->icsk_rto = inet_csk_rto_backoff(icsk, TCP_RTO_MAX);
440
441 tcp_mstamp_refresh(tp);
442 delta_us = (u32)(tp->tcp_mstamp - tcp_skb_timestamp_us(skb));
443 remaining = icsk->icsk_rto - usecs_to_jiffies(delta_us);
444
445 if (remaining > 0) {
446 inet_csk_reset_xmit_timer(sk, ICSK_TIME_RETRANS,
447 remaining, TCP_RTO_MAX);
448 } else {
449 /* RTO revert clocked out retransmission.
450 * Will retransmit now.
451 */
452 tcp_retransmit_timer(sk);
453 }
454 }
455 EXPORT_SYMBOL(tcp_ld_RTO_revert);
456
457 /*
458 * This routine is called by the ICMP module when it gets some
459 * sort of error condition. If err < 0 then the socket should
460 * be closed and the error returned to the user. If err > 0
461 * it's just the icmp type << 8 | icmp code. After adjustment
462 * header points to the first 8 bytes of the tcp header. We need
463 * to find the appropriate port.
464 *
465 * The locking strategy used here is very "optimistic". When
466 * someone else accesses the socket the ICMP is just dropped
467 * and for some paths there is no check at all.
468 * A more general error queue to queue errors for later handling
469 * is probably better.
470 *
471 */
472
tcp_v4_err(struct sk_buff * skb,u32 info)473 int tcp_v4_err(struct sk_buff *skb, u32 info)
474 {
475 const struct iphdr *iph = (const struct iphdr *)skb->data;
476 struct tcphdr *th = (struct tcphdr *)(skb->data + (iph->ihl << 2));
477 struct tcp_sock *tp;
478 struct inet_sock *inet;
479 const int type = icmp_hdr(skb)->type;
480 const int code = icmp_hdr(skb)->code;
481 struct sock *sk;
482 struct request_sock *fastopen;
483 u32 seq, snd_una;
484 int err;
485 struct net *net = dev_net(skb->dev);
486
487 sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
488 iph->daddr, th->dest, iph->saddr,
489 ntohs(th->source), inet_iif(skb), 0);
490 if (!sk) {
491 __ICMP_INC_STATS(net, ICMP_MIB_INERRORS);
492 return -ENOENT;
493 }
494 if (sk->sk_state == TCP_TIME_WAIT) {
495 inet_twsk_put(inet_twsk(sk));
496 return 0;
497 }
498 seq = ntohl(th->seq);
499 if (sk->sk_state == TCP_NEW_SYN_RECV) {
500 tcp_req_err(sk, seq, type == ICMP_PARAMETERPROB ||
501 type == ICMP_TIME_EXCEEDED ||
502 (type == ICMP_DEST_UNREACH &&
503 (code == ICMP_NET_UNREACH ||
504 code == ICMP_HOST_UNREACH)));
505 return 0;
506 }
507
508 bh_lock_sock(sk);
509 /* If too many ICMPs get dropped on busy
510 * servers this needs to be solved differently.
511 * We do take care of PMTU discovery (RFC1191) special case :
512 * we can receive locally generated ICMP messages while socket is held.
513 */
514 if (sock_owned_by_user(sk)) {
515 if (!(type == ICMP_DEST_UNREACH && code == ICMP_FRAG_NEEDED))
516 __NET_INC_STATS(net, LINUX_MIB_LOCKDROPPEDICMPS);
517 }
518 if (sk->sk_state == TCP_CLOSE)
519 goto out;
520
521 if (static_branch_unlikely(&ip4_min_ttl)) {
522 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
523 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
524 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
525 goto out;
526 }
527 }
528
529 tp = tcp_sk(sk);
530 /* XXX (TFO) - tp->snd_una should be ISN (tcp_create_openreq_child() */
531 fastopen = rcu_dereference(tp->fastopen_rsk);
532 snd_una = fastopen ? tcp_rsk(fastopen)->snt_isn : tp->snd_una;
533 if (sk->sk_state != TCP_LISTEN &&
534 !between(seq, snd_una, tp->snd_nxt)) {
535 __NET_INC_STATS(net, LINUX_MIB_OUTOFWINDOWICMPS);
536 goto out;
537 }
538
539 switch (type) {
540 case ICMP_REDIRECT:
541 if (!sock_owned_by_user(sk))
542 do_redirect(skb, sk);
543 goto out;
544 case ICMP_SOURCE_QUENCH:
545 /* Just silently ignore these. */
546 goto out;
547 case ICMP_PARAMETERPROB:
548 err = EPROTO;
549 break;
550 case ICMP_DEST_UNREACH:
551 if (code > NR_ICMP_UNREACH)
552 goto out;
553
554 if (code == ICMP_FRAG_NEEDED) { /* PMTU discovery (RFC1191) */
555 /* We are not interested in TCP_LISTEN and open_requests
556 * (SYN-ACKs send out by Linux are always <576bytes so
557 * they should go through unfragmented).
558 */
559 if (sk->sk_state == TCP_LISTEN)
560 goto out;
561
562 WRITE_ONCE(tp->mtu_info, info);
563 if (!sock_owned_by_user(sk)) {
564 tcp_v4_mtu_reduced(sk);
565 } else {
566 if (!test_and_set_bit(TCP_MTU_REDUCED_DEFERRED, &sk->sk_tsq_flags))
567 sock_hold(sk);
568 }
569 goto out;
570 }
571
572 err = icmp_err_convert[code].errno;
573 /* check if this ICMP message allows revert of backoff.
574 * (see RFC 6069)
575 */
576 if (!fastopen &&
577 (code == ICMP_NET_UNREACH || code == ICMP_HOST_UNREACH))
578 tcp_ld_RTO_revert(sk, seq);
579 break;
580 case ICMP_TIME_EXCEEDED:
581 err = EHOSTUNREACH;
582 break;
583 default:
584 goto out;
585 }
586
587 switch (sk->sk_state) {
588 case TCP_SYN_SENT:
589 case TCP_SYN_RECV:
590 /* Only in fast or simultaneous open. If a fast open socket is
591 * already accepted it is treated as a connected one below.
592 */
593 if (fastopen && !fastopen->sk)
594 break;
595
596 ip_icmp_error(sk, skb, err, th->dest, info, (u8 *)th);
597
598 if (!sock_owned_by_user(sk)) {
599 sk->sk_err = err;
600
601 sk_error_report(sk);
602
603 tcp_done(sk);
604 } else {
605 sk->sk_err_soft = err;
606 }
607 goto out;
608 }
609
610 /* If we've already connected we will keep trying
611 * until we time out, or the user gives up.
612 *
613 * rfc1122 4.2.3.9 allows to consider as hard errors
614 * only PROTO_UNREACH and PORT_UNREACH (well, FRAG_FAILED too,
615 * but it is obsoleted by pmtu discovery).
616 *
617 * Note, that in modern internet, where routing is unreliable
618 * and in each dark corner broken firewalls sit, sending random
619 * errors ordered by their masters even this two messages finally lose
620 * their original sense (even Linux sends invalid PORT_UNREACHs)
621 *
622 * Now we are in compliance with RFCs.
623 * --ANK (980905)
624 */
625
626 inet = inet_sk(sk);
627 if (!sock_owned_by_user(sk) && inet->recverr) {
628 sk->sk_err = err;
629 sk_error_report(sk);
630 } else { /* Only an error on timeout */
631 sk->sk_err_soft = err;
632 }
633
634 out:
635 bh_unlock_sock(sk);
636 sock_put(sk);
637 return 0;
638 }
639
__tcp_v4_send_check(struct sk_buff * skb,__be32 saddr,__be32 daddr)640 void __tcp_v4_send_check(struct sk_buff *skb, __be32 saddr, __be32 daddr)
641 {
642 struct tcphdr *th = tcp_hdr(skb);
643
644 th->check = ~tcp_v4_check(skb->len, saddr, daddr, 0);
645 skb->csum_start = skb_transport_header(skb) - skb->head;
646 skb->csum_offset = offsetof(struct tcphdr, check);
647 }
648
649 /* This routine computes an IPv4 TCP checksum. */
tcp_v4_send_check(struct sock * sk,struct sk_buff * skb)650 void tcp_v4_send_check(struct sock *sk, struct sk_buff *skb)
651 {
652 const struct inet_sock *inet = inet_sk(sk);
653
654 __tcp_v4_send_check(skb, inet->inet_saddr, inet->inet_daddr);
655 }
656 EXPORT_SYMBOL(tcp_v4_send_check);
657
658 /*
659 * This routine will send an RST to the other tcp.
660 *
661 * Someone asks: why I NEVER use socket parameters (TOS, TTL etc.)
662 * for reset.
663 * Answer: if a packet caused RST, it is not for a socket
664 * existing in our system, if it is matched to a socket,
665 * it is just duplicate segment or bug in other side's TCP.
666 * So that we build reply only basing on parameters
667 * arrived with segment.
668 * Exception: precedence violation. We do not implement it in any case.
669 */
670
671 #ifdef CONFIG_TCP_MD5SIG
672 #define OPTION_BYTES TCPOLEN_MD5SIG_ALIGNED
673 #else
674 #define OPTION_BYTES sizeof(__be32)
675 #endif
676
tcp_v4_send_reset(const struct sock * sk,struct sk_buff * skb)677 static void tcp_v4_send_reset(const struct sock *sk, struct sk_buff *skb)
678 {
679 const struct tcphdr *th = tcp_hdr(skb);
680 struct {
681 struct tcphdr th;
682 __be32 opt[OPTION_BYTES / sizeof(__be32)];
683 } rep;
684 struct ip_reply_arg arg;
685 #ifdef CONFIG_TCP_MD5SIG
686 struct tcp_md5sig_key *key = NULL;
687 const __u8 *hash_location = NULL;
688 unsigned char newhash[16];
689 int genhash;
690 struct sock *sk1 = NULL;
691 #endif
692 u64 transmit_time = 0;
693 struct sock *ctl_sk;
694 struct net *net;
695
696 /* Never send a reset in response to a reset. */
697 if (th->rst)
698 return;
699
700 /* If sk not NULL, it means we did a successful lookup and incoming
701 * route had to be correct. prequeue might have dropped our dst.
702 */
703 if (!sk && skb_rtable(skb)->rt_type != RTN_LOCAL)
704 return;
705
706 /* Swap the send and the receive. */
707 memset(&rep, 0, sizeof(rep));
708 rep.th.dest = th->source;
709 rep.th.source = th->dest;
710 rep.th.doff = sizeof(struct tcphdr) / 4;
711 rep.th.rst = 1;
712
713 if (th->ack) {
714 rep.th.seq = th->ack_seq;
715 } else {
716 rep.th.ack = 1;
717 rep.th.ack_seq = htonl(ntohl(th->seq) + th->syn + th->fin +
718 skb->len - (th->doff << 2));
719 }
720
721 memset(&arg, 0, sizeof(arg));
722 arg.iov[0].iov_base = (unsigned char *)&rep;
723 arg.iov[0].iov_len = sizeof(rep.th);
724
725 net = sk ? sock_net(sk) : dev_net(skb_dst(skb)->dev);
726 #ifdef CONFIG_TCP_MD5SIG
727 rcu_read_lock();
728 hash_location = tcp_parse_md5sig_option(th);
729 if (sk && sk_fullsock(sk)) {
730 const union tcp_md5_addr *addr;
731 int l3index;
732
733 /* sdif set, means packet ingressed via a device
734 * in an L3 domain and inet_iif is set to it.
735 */
736 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
737 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
738 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
739 } else if (hash_location) {
740 const union tcp_md5_addr *addr;
741 int sdif = tcp_v4_sdif(skb);
742 int dif = inet_iif(skb);
743 int l3index;
744
745 /*
746 * active side is lost. Try to find listening socket through
747 * source port, and then find md5 key through listening socket.
748 * we are not loose security here:
749 * Incoming packet is checked with md5 hash with finding key,
750 * no RST generated if md5 hash doesn't match.
751 */
752 sk1 = __inet_lookup_listener(net, net->ipv4.tcp_death_row.hashinfo,
753 NULL, 0, ip_hdr(skb)->saddr,
754 th->source, ip_hdr(skb)->daddr,
755 ntohs(th->source), dif, sdif);
756 /* don't send rst if it can't find key */
757 if (!sk1)
758 goto out;
759
760 /* sdif set, means packet ingressed via a device
761 * in an L3 domain and dif is set to it.
762 */
763 l3index = sdif ? dif : 0;
764 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
765 key = tcp_md5_do_lookup(sk1, l3index, addr, AF_INET);
766 if (!key)
767 goto out;
768
769
770 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
771 if (genhash || memcmp(hash_location, newhash, 16) != 0)
772 goto out;
773
774 }
775
776 if (key) {
777 rep.opt[0] = htonl((TCPOPT_NOP << 24) |
778 (TCPOPT_NOP << 16) |
779 (TCPOPT_MD5SIG << 8) |
780 TCPOLEN_MD5SIG);
781 /* Update length and the length the header thinks exists */
782 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
783 rep.th.doff = arg.iov[0].iov_len / 4;
784
785 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[1],
786 key, ip_hdr(skb)->saddr,
787 ip_hdr(skb)->daddr, &rep.th);
788 }
789 #endif
790 /* Can't co-exist with TCPMD5, hence check rep.opt[0] */
791 if (rep.opt[0] == 0) {
792 __be32 mrst = mptcp_reset_option(skb);
793
794 if (mrst) {
795 rep.opt[0] = mrst;
796 arg.iov[0].iov_len += sizeof(mrst);
797 rep.th.doff = arg.iov[0].iov_len / 4;
798 }
799 }
800
801 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
802 ip_hdr(skb)->saddr, /* XXX */
803 arg.iov[0].iov_len, IPPROTO_TCP, 0);
804 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
805 arg.flags = (sk && inet_sk_transparent(sk)) ? IP_REPLY_ARG_NOSRCCHECK : 0;
806
807 /* When socket is gone, all binding information is lost.
808 * routing might fail in this case. No choice here, if we choose to force
809 * input interface, we will misroute in case of asymmetric route.
810 */
811 if (sk) {
812 arg.bound_dev_if = sk->sk_bound_dev_if;
813 if (sk_fullsock(sk))
814 trace_tcp_send_reset(sk, skb);
815 }
816
817 BUILD_BUG_ON(offsetof(struct sock, sk_bound_dev_if) !=
818 offsetof(struct inet_timewait_sock, tw_bound_dev_if));
819
820 arg.tos = ip_hdr(skb)->tos;
821 arg.uid = sock_net_uid(net, sk && sk_fullsock(sk) ? sk : NULL);
822 local_bh_disable();
823 ctl_sk = this_cpu_read(ipv4_tcp_sk);
824 sock_net_set(ctl_sk, net);
825 if (sk) {
826 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
827 inet_twsk(sk)->tw_mark : sk->sk_mark;
828 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
829 inet_twsk(sk)->tw_priority : sk->sk_priority;
830 transmit_time = tcp_transmit_time(sk);
831 xfrm_sk_clone_policy(ctl_sk, sk);
832 }
833 ip_send_unicast_reply(ctl_sk,
834 skb, &TCP_SKB_CB(skb)->header.h4.opt,
835 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
836 &arg, arg.iov[0].iov_len,
837 transmit_time);
838
839 ctl_sk->sk_mark = 0;
840 xfrm_sk_free_policy(ctl_sk);
841 sock_net_set(ctl_sk, &init_net);
842 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
843 __TCP_INC_STATS(net, TCP_MIB_OUTRSTS);
844 local_bh_enable();
845
846 #ifdef CONFIG_TCP_MD5SIG
847 out:
848 rcu_read_unlock();
849 #endif
850 }
851
852 /* The code following below sending ACKs in SYN-RECV and TIME-WAIT states
853 outside socket context is ugly, certainly. What can I do?
854 */
855
tcp_v4_send_ack(const struct sock * sk,struct sk_buff * skb,u32 seq,u32 ack,u32 win,u32 tsval,u32 tsecr,int oif,struct tcp_md5sig_key * key,int reply_flags,u8 tos)856 static void tcp_v4_send_ack(const struct sock *sk,
857 struct sk_buff *skb, u32 seq, u32 ack,
858 u32 win, u32 tsval, u32 tsecr, int oif,
859 struct tcp_md5sig_key *key,
860 int reply_flags, u8 tos)
861 {
862 const struct tcphdr *th = tcp_hdr(skb);
863 struct {
864 struct tcphdr th;
865 __be32 opt[(TCPOLEN_TSTAMP_ALIGNED >> 2)
866 #ifdef CONFIG_TCP_MD5SIG
867 + (TCPOLEN_MD5SIG_ALIGNED >> 2)
868 #endif
869 ];
870 } rep;
871 struct net *net = sock_net(sk);
872 struct ip_reply_arg arg;
873 struct sock *ctl_sk;
874 u64 transmit_time;
875
876 memset(&rep.th, 0, sizeof(struct tcphdr));
877 memset(&arg, 0, sizeof(arg));
878
879 arg.iov[0].iov_base = (unsigned char *)&rep;
880 arg.iov[0].iov_len = sizeof(rep.th);
881 if (tsecr) {
882 rep.opt[0] = htonl((TCPOPT_NOP << 24) | (TCPOPT_NOP << 16) |
883 (TCPOPT_TIMESTAMP << 8) |
884 TCPOLEN_TIMESTAMP);
885 rep.opt[1] = htonl(tsval);
886 rep.opt[2] = htonl(tsecr);
887 arg.iov[0].iov_len += TCPOLEN_TSTAMP_ALIGNED;
888 }
889
890 /* Swap the send and the receive. */
891 rep.th.dest = th->source;
892 rep.th.source = th->dest;
893 rep.th.doff = arg.iov[0].iov_len / 4;
894 rep.th.seq = htonl(seq);
895 rep.th.ack_seq = htonl(ack);
896 rep.th.ack = 1;
897 rep.th.window = htons(win);
898
899 #ifdef CONFIG_TCP_MD5SIG
900 if (key) {
901 int offset = (tsecr) ? 3 : 0;
902
903 rep.opt[offset++] = htonl((TCPOPT_NOP << 24) |
904 (TCPOPT_NOP << 16) |
905 (TCPOPT_MD5SIG << 8) |
906 TCPOLEN_MD5SIG);
907 arg.iov[0].iov_len += TCPOLEN_MD5SIG_ALIGNED;
908 rep.th.doff = arg.iov[0].iov_len/4;
909
910 tcp_v4_md5_hash_hdr((__u8 *) &rep.opt[offset],
911 key, ip_hdr(skb)->saddr,
912 ip_hdr(skb)->daddr, &rep.th);
913 }
914 #endif
915 arg.flags = reply_flags;
916 arg.csum = csum_tcpudp_nofold(ip_hdr(skb)->daddr,
917 ip_hdr(skb)->saddr, /* XXX */
918 arg.iov[0].iov_len, IPPROTO_TCP, 0);
919 arg.csumoffset = offsetof(struct tcphdr, check) / 2;
920 if (oif)
921 arg.bound_dev_if = oif;
922 arg.tos = tos;
923 arg.uid = sock_net_uid(net, sk_fullsock(sk) ? sk : NULL);
924 local_bh_disable();
925 ctl_sk = this_cpu_read(ipv4_tcp_sk);
926 sock_net_set(ctl_sk, net);
927 ctl_sk->sk_mark = (sk->sk_state == TCP_TIME_WAIT) ?
928 inet_twsk(sk)->tw_mark : sk->sk_mark;
929 ctl_sk->sk_priority = (sk->sk_state == TCP_TIME_WAIT) ?
930 inet_twsk(sk)->tw_priority : sk->sk_priority;
931 transmit_time = tcp_transmit_time(sk);
932 ip_send_unicast_reply(ctl_sk,
933 skb, &TCP_SKB_CB(skb)->header.h4.opt,
934 ip_hdr(skb)->saddr, ip_hdr(skb)->daddr,
935 &arg, arg.iov[0].iov_len,
936 transmit_time);
937
938 ctl_sk->sk_mark = 0;
939 sock_net_set(ctl_sk, &init_net);
940 __TCP_INC_STATS(net, TCP_MIB_OUTSEGS);
941 local_bh_enable();
942 }
943
tcp_v4_timewait_ack(struct sock * sk,struct sk_buff * skb)944 static void tcp_v4_timewait_ack(struct sock *sk, struct sk_buff *skb)
945 {
946 struct inet_timewait_sock *tw = inet_twsk(sk);
947 struct tcp_timewait_sock *tcptw = tcp_twsk(sk);
948
949 tcp_v4_send_ack(sk, skb,
950 tcptw->tw_snd_nxt, tcptw->tw_rcv_nxt,
951 tcptw->tw_rcv_wnd >> tw->tw_rcv_wscale,
952 tcp_time_stamp_raw() + tcptw->tw_ts_offset,
953 tcptw->tw_ts_recent,
954 tw->tw_bound_dev_if,
955 tcp_twsk_md5_key(tcptw),
956 tw->tw_transparent ? IP_REPLY_ARG_NOSRCCHECK : 0,
957 tw->tw_tos
958 );
959
960 inet_twsk_put(tw);
961 }
962
tcp_v4_reqsk_send_ack(const struct sock * sk,struct sk_buff * skb,struct request_sock * req)963 static void tcp_v4_reqsk_send_ack(const struct sock *sk, struct sk_buff *skb,
964 struct request_sock *req)
965 {
966 const union tcp_md5_addr *addr;
967 int l3index;
968
969 /* sk->sk_state == TCP_LISTEN -> for regular TCP_SYN_RECV
970 * sk->sk_state == TCP_SYN_RECV -> for Fast Open.
971 */
972 u32 seq = (sk->sk_state == TCP_LISTEN) ? tcp_rsk(req)->snt_isn + 1 :
973 tcp_sk(sk)->snd_nxt;
974
975 /* RFC 7323 2.3
976 * The window field (SEG.WND) of every outgoing segment, with the
977 * exception of <SYN> segments, MUST be right-shifted by
978 * Rcv.Wind.Shift bits:
979 */
980 addr = (union tcp_md5_addr *)&ip_hdr(skb)->saddr;
981 l3index = tcp_v4_sdif(skb) ? inet_iif(skb) : 0;
982 tcp_v4_send_ack(sk, skb, seq,
983 tcp_rsk(req)->rcv_nxt,
984 req->rsk_rcv_wnd >> inet_rsk(req)->rcv_wscale,
985 tcp_time_stamp_raw() + tcp_rsk(req)->ts_off,
986 req->ts_recent,
987 0,
988 tcp_md5_do_lookup(sk, l3index, addr, AF_INET),
989 inet_rsk(req)->no_srccheck ? IP_REPLY_ARG_NOSRCCHECK : 0,
990 ip_hdr(skb)->tos);
991 }
992
993 /*
994 * Send a SYN-ACK after having received a SYN.
995 * This still operates on a request_sock only, not on a big
996 * socket.
997 */
tcp_v4_send_synack(const struct sock * sk,struct dst_entry * dst,struct flowi * fl,struct request_sock * req,struct tcp_fastopen_cookie * foc,enum tcp_synack_type synack_type,struct sk_buff * syn_skb)998 static int tcp_v4_send_synack(const struct sock *sk, struct dst_entry *dst,
999 struct flowi *fl,
1000 struct request_sock *req,
1001 struct tcp_fastopen_cookie *foc,
1002 enum tcp_synack_type synack_type,
1003 struct sk_buff *syn_skb)
1004 {
1005 const struct inet_request_sock *ireq = inet_rsk(req);
1006 struct flowi4 fl4;
1007 int err = -1;
1008 struct sk_buff *skb;
1009 u8 tos;
1010
1011 /* First, grab a route. */
1012 if (!dst && (dst = inet_csk_route_req(sk, &fl4, req)) == NULL)
1013 return -1;
1014
1015 skb = tcp_make_synack(sk, dst, req, foc, synack_type, syn_skb);
1016
1017 if (skb) {
1018 __tcp_v4_send_check(skb, ireq->ir_loc_addr, ireq->ir_rmt_addr);
1019
1020 tos = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos) ?
1021 (tcp_rsk(req)->syn_tos & ~INET_ECN_MASK) |
1022 (inet_sk(sk)->tos & INET_ECN_MASK) :
1023 inet_sk(sk)->tos;
1024
1025 if (!INET_ECN_is_capable(tos) &&
1026 tcp_bpf_ca_needs_ecn((struct sock *)req))
1027 tos |= INET_ECN_ECT_0;
1028
1029 rcu_read_lock();
1030 err = ip_build_and_send_pkt(skb, sk, ireq->ir_loc_addr,
1031 ireq->ir_rmt_addr,
1032 rcu_dereference(ireq->ireq_opt),
1033 tos);
1034 rcu_read_unlock();
1035 err = net_xmit_eval(err);
1036 }
1037
1038 return err;
1039 }
1040
1041 /*
1042 * IPv4 request_sock destructor.
1043 */
tcp_v4_reqsk_destructor(struct request_sock * req)1044 static void tcp_v4_reqsk_destructor(struct request_sock *req)
1045 {
1046 kfree(rcu_dereference_protected(inet_rsk(req)->ireq_opt, 1));
1047 }
1048
1049 #ifdef CONFIG_TCP_MD5SIG
1050 /*
1051 * RFC2385 MD5 checksumming requires a mapping of
1052 * IP address->MD5 Key.
1053 * We need to maintain these in the sk structure.
1054 */
1055
1056 DEFINE_STATIC_KEY_FALSE(tcp_md5_needed);
1057 EXPORT_SYMBOL(tcp_md5_needed);
1058
better_md5_match(struct tcp_md5sig_key * old,struct tcp_md5sig_key * new)1059 static bool better_md5_match(struct tcp_md5sig_key *old, struct tcp_md5sig_key *new)
1060 {
1061 if (!old)
1062 return true;
1063
1064 /* l3index always overrides non-l3index */
1065 if (old->l3index && new->l3index == 0)
1066 return false;
1067 if (old->l3index == 0 && new->l3index)
1068 return true;
1069
1070 return old->prefixlen < new->prefixlen;
1071 }
1072
1073 /* Find the Key structure for an address. */
__tcp_md5_do_lookup(const struct sock * sk,int l3index,const union tcp_md5_addr * addr,int family)1074 struct tcp_md5sig_key *__tcp_md5_do_lookup(const struct sock *sk, int l3index,
1075 const union tcp_md5_addr *addr,
1076 int family)
1077 {
1078 const struct tcp_sock *tp = tcp_sk(sk);
1079 struct tcp_md5sig_key *key;
1080 const struct tcp_md5sig_info *md5sig;
1081 __be32 mask;
1082 struct tcp_md5sig_key *best_match = NULL;
1083 bool match;
1084
1085 /* caller either holds rcu_read_lock() or socket lock */
1086 md5sig = rcu_dereference_check(tp->md5sig_info,
1087 lockdep_sock_is_held(sk));
1088 if (!md5sig)
1089 return NULL;
1090
1091 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1092 lockdep_sock_is_held(sk)) {
1093 if (key->family != family)
1094 continue;
1095 if (key->flags & TCP_MD5SIG_FLAG_IFINDEX && key->l3index != l3index)
1096 continue;
1097 if (family == AF_INET) {
1098 mask = inet_make_mask(key->prefixlen);
1099 match = (key->addr.a4.s_addr & mask) ==
1100 (addr->a4.s_addr & mask);
1101 #if IS_ENABLED(CONFIG_IPV6)
1102 } else if (family == AF_INET6) {
1103 match = ipv6_prefix_equal(&key->addr.a6, &addr->a6,
1104 key->prefixlen);
1105 #endif
1106 } else {
1107 match = false;
1108 }
1109
1110 if (match && better_md5_match(best_match, key))
1111 best_match = key;
1112 }
1113 return best_match;
1114 }
1115 EXPORT_SYMBOL(__tcp_md5_do_lookup);
1116
tcp_md5_do_lookup_exact(const struct sock * sk,const union tcp_md5_addr * addr,int family,u8 prefixlen,int l3index,u8 flags)1117 static struct tcp_md5sig_key *tcp_md5_do_lookup_exact(const struct sock *sk,
1118 const union tcp_md5_addr *addr,
1119 int family, u8 prefixlen,
1120 int l3index, u8 flags)
1121 {
1122 const struct tcp_sock *tp = tcp_sk(sk);
1123 struct tcp_md5sig_key *key;
1124 unsigned int size = sizeof(struct in_addr);
1125 const struct tcp_md5sig_info *md5sig;
1126
1127 /* caller either holds rcu_read_lock() or socket lock */
1128 md5sig = rcu_dereference_check(tp->md5sig_info,
1129 lockdep_sock_is_held(sk));
1130 if (!md5sig)
1131 return NULL;
1132 #if IS_ENABLED(CONFIG_IPV6)
1133 if (family == AF_INET6)
1134 size = sizeof(struct in6_addr);
1135 #endif
1136 hlist_for_each_entry_rcu(key, &md5sig->head, node,
1137 lockdep_sock_is_held(sk)) {
1138 if (key->family != family)
1139 continue;
1140 if ((key->flags & TCP_MD5SIG_FLAG_IFINDEX) != (flags & TCP_MD5SIG_FLAG_IFINDEX))
1141 continue;
1142 if (key->l3index != l3index)
1143 continue;
1144 if (!memcmp(&key->addr, addr, size) &&
1145 key->prefixlen == prefixlen)
1146 return key;
1147 }
1148 return NULL;
1149 }
1150
tcp_v4_md5_lookup(const struct sock * sk,const struct sock * addr_sk)1151 struct tcp_md5sig_key *tcp_v4_md5_lookup(const struct sock *sk,
1152 const struct sock *addr_sk)
1153 {
1154 const union tcp_md5_addr *addr;
1155 int l3index;
1156
1157 l3index = l3mdev_master_ifindex_by_index(sock_net(sk),
1158 addr_sk->sk_bound_dev_if);
1159 addr = (const union tcp_md5_addr *)&addr_sk->sk_daddr;
1160 return tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1161 }
1162 EXPORT_SYMBOL(tcp_v4_md5_lookup);
1163
1164 /* This can be called on a newly created socket, from other files */
tcp_md5_do_add(struct sock * sk,const union tcp_md5_addr * addr,int family,u8 prefixlen,int l3index,u8 flags,const u8 * newkey,u8 newkeylen,gfp_t gfp)1165 int tcp_md5_do_add(struct sock *sk, const union tcp_md5_addr *addr,
1166 int family, u8 prefixlen, int l3index, u8 flags,
1167 const u8 *newkey, u8 newkeylen, gfp_t gfp)
1168 {
1169 /* Add Key to the list */
1170 struct tcp_md5sig_key *key;
1171 struct tcp_sock *tp = tcp_sk(sk);
1172 struct tcp_md5sig_info *md5sig;
1173
1174 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1175 if (key) {
1176 /* Pre-existing entry - just update that one.
1177 * Note that the key might be used concurrently.
1178 * data_race() is telling kcsan that we do not care of
1179 * key mismatches, since changing MD5 key on live flows
1180 * can lead to packet drops.
1181 */
1182 data_race(memcpy(key->key, newkey, newkeylen));
1183
1184 /* Pairs with READ_ONCE() in tcp_md5_hash_key().
1185 * Also note that a reader could catch new key->keylen value
1186 * but old key->key[], this is the reason we use __GFP_ZERO
1187 * at sock_kmalloc() time below these lines.
1188 */
1189 WRITE_ONCE(key->keylen, newkeylen);
1190
1191 return 0;
1192 }
1193
1194 md5sig = rcu_dereference_protected(tp->md5sig_info,
1195 lockdep_sock_is_held(sk));
1196 if (!md5sig) {
1197 md5sig = kmalloc(sizeof(*md5sig), gfp);
1198 if (!md5sig)
1199 return -ENOMEM;
1200
1201 sk_gso_disable(sk);
1202 INIT_HLIST_HEAD(&md5sig->head);
1203 rcu_assign_pointer(tp->md5sig_info, md5sig);
1204 }
1205
1206 key = sock_kmalloc(sk, sizeof(*key), gfp | __GFP_ZERO);
1207 if (!key)
1208 return -ENOMEM;
1209 if (!tcp_alloc_md5sig_pool()) {
1210 sock_kfree_s(sk, key, sizeof(*key));
1211 return -ENOMEM;
1212 }
1213
1214 memcpy(key->key, newkey, newkeylen);
1215 key->keylen = newkeylen;
1216 key->family = family;
1217 key->prefixlen = prefixlen;
1218 key->l3index = l3index;
1219 key->flags = flags;
1220 memcpy(&key->addr, addr,
1221 (IS_ENABLED(CONFIG_IPV6) && family == AF_INET6) ? sizeof(struct in6_addr) :
1222 sizeof(struct in_addr));
1223 hlist_add_head_rcu(&key->node, &md5sig->head);
1224 return 0;
1225 }
1226 EXPORT_SYMBOL(tcp_md5_do_add);
1227
tcp_md5_do_del(struct sock * sk,const union tcp_md5_addr * addr,int family,u8 prefixlen,int l3index,u8 flags)1228 int tcp_md5_do_del(struct sock *sk, const union tcp_md5_addr *addr, int family,
1229 u8 prefixlen, int l3index, u8 flags)
1230 {
1231 struct tcp_md5sig_key *key;
1232
1233 key = tcp_md5_do_lookup_exact(sk, addr, family, prefixlen, l3index, flags);
1234 if (!key)
1235 return -ENOENT;
1236 hlist_del_rcu(&key->node);
1237 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1238 kfree_rcu(key, rcu);
1239 return 0;
1240 }
1241 EXPORT_SYMBOL(tcp_md5_do_del);
1242
tcp_clear_md5_list(struct sock * sk)1243 static void tcp_clear_md5_list(struct sock *sk)
1244 {
1245 struct tcp_sock *tp = tcp_sk(sk);
1246 struct tcp_md5sig_key *key;
1247 struct hlist_node *n;
1248 struct tcp_md5sig_info *md5sig;
1249
1250 md5sig = rcu_dereference_protected(tp->md5sig_info, 1);
1251
1252 hlist_for_each_entry_safe(key, n, &md5sig->head, node) {
1253 hlist_del_rcu(&key->node);
1254 atomic_sub(sizeof(*key), &sk->sk_omem_alloc);
1255 kfree_rcu(key, rcu);
1256 }
1257 }
1258
tcp_v4_parse_md5_keys(struct sock * sk,int optname,sockptr_t optval,int optlen)1259 static int tcp_v4_parse_md5_keys(struct sock *sk, int optname,
1260 sockptr_t optval, int optlen)
1261 {
1262 struct tcp_md5sig cmd;
1263 struct sockaddr_in *sin = (struct sockaddr_in *)&cmd.tcpm_addr;
1264 const union tcp_md5_addr *addr;
1265 u8 prefixlen = 32;
1266 int l3index = 0;
1267 u8 flags;
1268
1269 if (optlen < sizeof(cmd))
1270 return -EINVAL;
1271
1272 if (copy_from_sockptr(&cmd, optval, sizeof(cmd)))
1273 return -EFAULT;
1274
1275 if (sin->sin_family != AF_INET)
1276 return -EINVAL;
1277
1278 flags = cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX;
1279
1280 if (optname == TCP_MD5SIG_EXT &&
1281 cmd.tcpm_flags & TCP_MD5SIG_FLAG_PREFIX) {
1282 prefixlen = cmd.tcpm_prefixlen;
1283 if (prefixlen > 32)
1284 return -EINVAL;
1285 }
1286
1287 if (optname == TCP_MD5SIG_EXT && cmd.tcpm_ifindex &&
1288 cmd.tcpm_flags & TCP_MD5SIG_FLAG_IFINDEX) {
1289 struct net_device *dev;
1290
1291 rcu_read_lock();
1292 dev = dev_get_by_index_rcu(sock_net(sk), cmd.tcpm_ifindex);
1293 if (dev && netif_is_l3_master(dev))
1294 l3index = dev->ifindex;
1295
1296 rcu_read_unlock();
1297
1298 /* ok to reference set/not set outside of rcu;
1299 * right now device MUST be an L3 master
1300 */
1301 if (!dev || !l3index)
1302 return -EINVAL;
1303 }
1304
1305 addr = (union tcp_md5_addr *)&sin->sin_addr.s_addr;
1306
1307 if (!cmd.tcpm_keylen)
1308 return tcp_md5_do_del(sk, addr, AF_INET, prefixlen, l3index, flags);
1309
1310 if (cmd.tcpm_keylen > TCP_MD5SIG_MAXKEYLEN)
1311 return -EINVAL;
1312
1313 return tcp_md5_do_add(sk, addr, AF_INET, prefixlen, l3index, flags,
1314 cmd.tcpm_key, cmd.tcpm_keylen, GFP_KERNEL);
1315 }
1316
tcp_v4_md5_hash_headers(struct tcp_md5sig_pool * hp,__be32 daddr,__be32 saddr,const struct tcphdr * th,int nbytes)1317 static int tcp_v4_md5_hash_headers(struct tcp_md5sig_pool *hp,
1318 __be32 daddr, __be32 saddr,
1319 const struct tcphdr *th, int nbytes)
1320 {
1321 struct tcp4_pseudohdr *bp;
1322 struct scatterlist sg;
1323 struct tcphdr *_th;
1324
1325 bp = hp->scratch;
1326 bp->saddr = saddr;
1327 bp->daddr = daddr;
1328 bp->pad = 0;
1329 bp->protocol = IPPROTO_TCP;
1330 bp->len = cpu_to_be16(nbytes);
1331
1332 _th = (struct tcphdr *)(bp + 1);
1333 memcpy(_th, th, sizeof(*th));
1334 _th->check = 0;
1335
1336 sg_init_one(&sg, bp, sizeof(*bp) + sizeof(*th));
1337 ahash_request_set_crypt(hp->md5_req, &sg, NULL,
1338 sizeof(*bp) + sizeof(*th));
1339 return crypto_ahash_update(hp->md5_req);
1340 }
1341
tcp_v4_md5_hash_hdr(char * md5_hash,const struct tcp_md5sig_key * key,__be32 daddr,__be32 saddr,const struct tcphdr * th)1342 static int tcp_v4_md5_hash_hdr(char *md5_hash, const struct tcp_md5sig_key *key,
1343 __be32 daddr, __be32 saddr, const struct tcphdr *th)
1344 {
1345 struct tcp_md5sig_pool *hp;
1346 struct ahash_request *req;
1347
1348 hp = tcp_get_md5sig_pool();
1349 if (!hp)
1350 goto clear_hash_noput;
1351 req = hp->md5_req;
1352
1353 if (crypto_ahash_init(req))
1354 goto clear_hash;
1355 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, th->doff << 2))
1356 goto clear_hash;
1357 if (tcp_md5_hash_key(hp, key))
1358 goto clear_hash;
1359 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1360 if (crypto_ahash_final(req))
1361 goto clear_hash;
1362
1363 tcp_put_md5sig_pool();
1364 return 0;
1365
1366 clear_hash:
1367 tcp_put_md5sig_pool();
1368 clear_hash_noput:
1369 memset(md5_hash, 0, 16);
1370 return 1;
1371 }
1372
tcp_v4_md5_hash_skb(char * md5_hash,const struct tcp_md5sig_key * key,const struct sock * sk,const struct sk_buff * skb)1373 int tcp_v4_md5_hash_skb(char *md5_hash, const struct tcp_md5sig_key *key,
1374 const struct sock *sk,
1375 const struct sk_buff *skb)
1376 {
1377 struct tcp_md5sig_pool *hp;
1378 struct ahash_request *req;
1379 const struct tcphdr *th = tcp_hdr(skb);
1380 __be32 saddr, daddr;
1381
1382 if (sk) { /* valid for establish/request sockets */
1383 saddr = sk->sk_rcv_saddr;
1384 daddr = sk->sk_daddr;
1385 } else {
1386 const struct iphdr *iph = ip_hdr(skb);
1387 saddr = iph->saddr;
1388 daddr = iph->daddr;
1389 }
1390
1391 hp = tcp_get_md5sig_pool();
1392 if (!hp)
1393 goto clear_hash_noput;
1394 req = hp->md5_req;
1395
1396 if (crypto_ahash_init(req))
1397 goto clear_hash;
1398
1399 if (tcp_v4_md5_hash_headers(hp, daddr, saddr, th, skb->len))
1400 goto clear_hash;
1401 if (tcp_md5_hash_skb_data(hp, skb, th->doff << 2))
1402 goto clear_hash;
1403 if (tcp_md5_hash_key(hp, key))
1404 goto clear_hash;
1405 ahash_request_set_crypt(req, NULL, md5_hash, 0);
1406 if (crypto_ahash_final(req))
1407 goto clear_hash;
1408
1409 tcp_put_md5sig_pool();
1410 return 0;
1411
1412 clear_hash:
1413 tcp_put_md5sig_pool();
1414 clear_hash_noput:
1415 memset(md5_hash, 0, 16);
1416 return 1;
1417 }
1418 EXPORT_SYMBOL(tcp_v4_md5_hash_skb);
1419
1420 #endif
1421
tcp_v4_init_req(struct request_sock * req,const struct sock * sk_listener,struct sk_buff * skb)1422 static void tcp_v4_init_req(struct request_sock *req,
1423 const struct sock *sk_listener,
1424 struct sk_buff *skb)
1425 {
1426 struct inet_request_sock *ireq = inet_rsk(req);
1427 struct net *net = sock_net(sk_listener);
1428
1429 sk_rcv_saddr_set(req_to_sk(req), ip_hdr(skb)->daddr);
1430 sk_daddr_set(req_to_sk(req), ip_hdr(skb)->saddr);
1431 RCU_INIT_POINTER(ireq->ireq_opt, tcp_v4_save_options(net, skb));
1432 }
1433
tcp_v4_route_req(const struct sock * sk,struct sk_buff * skb,struct flowi * fl,struct request_sock * req)1434 static struct dst_entry *tcp_v4_route_req(const struct sock *sk,
1435 struct sk_buff *skb,
1436 struct flowi *fl,
1437 struct request_sock *req)
1438 {
1439 tcp_v4_init_req(req, sk, skb);
1440
1441 if (security_inet_conn_request(sk, skb, req))
1442 return NULL;
1443
1444 return inet_csk_route_req(sk, &fl->u.ip4, req);
1445 }
1446
1447 struct request_sock_ops tcp_request_sock_ops __read_mostly = {
1448 .family = PF_INET,
1449 .obj_size = sizeof(struct tcp_request_sock),
1450 .rtx_syn_ack = tcp_rtx_synack,
1451 .send_ack = tcp_v4_reqsk_send_ack,
1452 .destructor = tcp_v4_reqsk_destructor,
1453 .send_reset = tcp_v4_send_reset,
1454 .syn_ack_timeout = tcp_syn_ack_timeout,
1455 };
1456
1457 const struct tcp_request_sock_ops tcp_request_sock_ipv4_ops = {
1458 .mss_clamp = TCP_MSS_DEFAULT,
1459 #ifdef CONFIG_TCP_MD5SIG
1460 .req_md5_lookup = tcp_v4_md5_lookup,
1461 .calc_md5_hash = tcp_v4_md5_hash_skb,
1462 #endif
1463 #ifdef CONFIG_SYN_COOKIES
1464 .cookie_init_seq = cookie_v4_init_sequence,
1465 #endif
1466 .route_req = tcp_v4_route_req,
1467 .init_seq = tcp_v4_init_seq,
1468 .init_ts_off = tcp_v4_init_ts_off,
1469 .send_synack = tcp_v4_send_synack,
1470 };
1471
tcp_v4_conn_request(struct sock * sk,struct sk_buff * skb)1472 int tcp_v4_conn_request(struct sock *sk, struct sk_buff *skb)
1473 {
1474 /* Never answer to SYNs send to broadcast or multicast */
1475 if (skb_rtable(skb)->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST))
1476 goto drop;
1477
1478 return tcp_conn_request(&tcp_request_sock_ops,
1479 &tcp_request_sock_ipv4_ops, sk, skb);
1480
1481 drop:
1482 tcp_listendrop(sk);
1483 return 0;
1484 }
1485 EXPORT_SYMBOL(tcp_v4_conn_request);
1486
1487
1488 /*
1489 * The three way handshake has completed - we got a valid synack -
1490 * now create the new socket.
1491 */
tcp_v4_syn_recv_sock(const struct sock * sk,struct sk_buff * skb,struct request_sock * req,struct dst_entry * dst,struct request_sock * req_unhash,bool * own_req)1492 struct sock *tcp_v4_syn_recv_sock(const struct sock *sk, struct sk_buff *skb,
1493 struct request_sock *req,
1494 struct dst_entry *dst,
1495 struct request_sock *req_unhash,
1496 bool *own_req)
1497 {
1498 struct inet_request_sock *ireq;
1499 bool found_dup_sk = false;
1500 struct inet_sock *newinet;
1501 struct tcp_sock *newtp;
1502 struct sock *newsk;
1503 #ifdef CONFIG_TCP_MD5SIG
1504 const union tcp_md5_addr *addr;
1505 struct tcp_md5sig_key *key;
1506 int l3index;
1507 #endif
1508 struct ip_options_rcu *inet_opt;
1509
1510 if (sk_acceptq_is_full(sk))
1511 goto exit_overflow;
1512
1513 newsk = tcp_create_openreq_child(sk, req, skb);
1514 if (!newsk)
1515 goto exit_nonewsk;
1516
1517 newsk->sk_gso_type = SKB_GSO_TCPV4;
1518 inet_sk_rx_dst_set(newsk, skb);
1519
1520 newtp = tcp_sk(newsk);
1521 newinet = inet_sk(newsk);
1522 ireq = inet_rsk(req);
1523 sk_daddr_set(newsk, ireq->ir_rmt_addr);
1524 sk_rcv_saddr_set(newsk, ireq->ir_loc_addr);
1525 newsk->sk_bound_dev_if = ireq->ir_iif;
1526 newinet->inet_saddr = ireq->ir_loc_addr;
1527 inet_opt = rcu_dereference(ireq->ireq_opt);
1528 RCU_INIT_POINTER(newinet->inet_opt, inet_opt);
1529 newinet->mc_index = inet_iif(skb);
1530 newinet->mc_ttl = ip_hdr(skb)->ttl;
1531 newinet->rcv_tos = ip_hdr(skb)->tos;
1532 inet_csk(newsk)->icsk_ext_hdr_len = 0;
1533 if (inet_opt)
1534 inet_csk(newsk)->icsk_ext_hdr_len = inet_opt->opt.optlen;
1535 newinet->inet_id = get_random_u16();
1536
1537 /* Set ToS of the new socket based upon the value of incoming SYN.
1538 * ECT bits are set later in tcp_init_transfer().
1539 */
1540 if (READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reflect_tos))
1541 newinet->tos = tcp_rsk(req)->syn_tos & ~INET_ECN_MASK;
1542
1543 if (!dst) {
1544 dst = inet_csk_route_child_sock(sk, newsk, req);
1545 if (!dst)
1546 goto put_and_exit;
1547 } else {
1548 /* syncookie case : see end of cookie_v4_check() */
1549 }
1550 sk_setup_caps(newsk, dst);
1551
1552 tcp_ca_openreq_child(newsk, dst);
1553
1554 tcp_sync_mss(newsk, dst_mtu(dst));
1555 newtp->advmss = tcp_mss_clamp(tcp_sk(sk), dst_metric_advmss(dst));
1556
1557 tcp_initialize_rcv_mss(newsk);
1558
1559 #ifdef CONFIG_TCP_MD5SIG
1560 l3index = l3mdev_master_ifindex_by_index(sock_net(sk), ireq->ir_iif);
1561 /* Copy over the MD5 key from the original socket */
1562 addr = (union tcp_md5_addr *)&newinet->inet_daddr;
1563 key = tcp_md5_do_lookup(sk, l3index, addr, AF_INET);
1564 if (key) {
1565 /*
1566 * We're using one, so create a matching key
1567 * on the newsk structure. If we fail to get
1568 * memory, then we end up not copying the key
1569 * across. Shucks.
1570 */
1571 tcp_md5_do_add(newsk, addr, AF_INET, 32, l3index, key->flags,
1572 key->key, key->keylen, GFP_ATOMIC);
1573 sk_gso_disable(newsk);
1574 }
1575 #endif
1576
1577 if (__inet_inherit_port(sk, newsk) < 0)
1578 goto put_and_exit;
1579 *own_req = inet_ehash_nolisten(newsk, req_to_sk(req_unhash),
1580 &found_dup_sk);
1581 if (likely(*own_req)) {
1582 tcp_move_syn(newtp, req);
1583 ireq->ireq_opt = NULL;
1584 } else {
1585 newinet->inet_opt = NULL;
1586
1587 if (!req_unhash && found_dup_sk) {
1588 /* This code path should only be executed in the
1589 * syncookie case only
1590 */
1591 bh_unlock_sock(newsk);
1592 sock_put(newsk);
1593 newsk = NULL;
1594 }
1595 }
1596 return newsk;
1597
1598 exit_overflow:
1599 NET_INC_STATS(sock_net(sk), LINUX_MIB_LISTENOVERFLOWS);
1600 exit_nonewsk:
1601 dst_release(dst);
1602 exit:
1603 tcp_listendrop(sk);
1604 return NULL;
1605 put_and_exit:
1606 newinet->inet_opt = NULL;
1607 inet_csk_prepare_forced_close(newsk);
1608 tcp_done(newsk);
1609 goto exit;
1610 }
1611 EXPORT_SYMBOL(tcp_v4_syn_recv_sock);
1612
tcp_v4_cookie_check(struct sock * sk,struct sk_buff * skb)1613 static struct sock *tcp_v4_cookie_check(struct sock *sk, struct sk_buff *skb)
1614 {
1615 #ifdef CONFIG_SYN_COOKIES
1616 const struct tcphdr *th = tcp_hdr(skb);
1617
1618 if (!th->syn)
1619 sk = cookie_v4_check(sk, skb);
1620 #endif
1621 return sk;
1622 }
1623
tcp_v4_get_syncookie(struct sock * sk,struct iphdr * iph,struct tcphdr * th,u32 * cookie)1624 u16 tcp_v4_get_syncookie(struct sock *sk, struct iphdr *iph,
1625 struct tcphdr *th, u32 *cookie)
1626 {
1627 u16 mss = 0;
1628 #ifdef CONFIG_SYN_COOKIES
1629 mss = tcp_get_syncookie_mss(&tcp_request_sock_ops,
1630 &tcp_request_sock_ipv4_ops, sk, th);
1631 if (mss) {
1632 *cookie = __cookie_v4_init_sequence(iph, th, &mss);
1633 tcp_synq_overflow(sk);
1634 }
1635 #endif
1636 return mss;
1637 }
1638
1639 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
1640 u32));
1641 /* The socket must have it's spinlock held when we get
1642 * here, unless it is a TCP_LISTEN socket.
1643 *
1644 * We have a potential double-lock case here, so even when
1645 * doing backlog processing we use the BH locking scheme.
1646 * This is because we cannot sleep with the original spinlock
1647 * held.
1648 */
tcp_v4_do_rcv(struct sock * sk,struct sk_buff * skb)1649 int tcp_v4_do_rcv(struct sock *sk, struct sk_buff *skb)
1650 {
1651 enum skb_drop_reason reason;
1652 struct sock *rsk;
1653
1654 if (sk->sk_state == TCP_ESTABLISHED) { /* Fast path */
1655 struct dst_entry *dst;
1656
1657 dst = rcu_dereference_protected(sk->sk_rx_dst,
1658 lockdep_sock_is_held(sk));
1659
1660 sock_rps_save_rxhash(sk, skb);
1661 sk_mark_napi_id(sk, skb);
1662 if (dst) {
1663 if (sk->sk_rx_dst_ifindex != skb->skb_iif ||
1664 !INDIRECT_CALL_1(dst->ops->check, ipv4_dst_check,
1665 dst, 0)) {
1666 RCU_INIT_POINTER(sk->sk_rx_dst, NULL);
1667 dst_release(dst);
1668 }
1669 }
1670 tcp_rcv_established(sk, skb);
1671 return 0;
1672 }
1673
1674 reason = SKB_DROP_REASON_NOT_SPECIFIED;
1675 if (tcp_checksum_complete(skb))
1676 goto csum_err;
1677
1678 if (sk->sk_state == TCP_LISTEN) {
1679 struct sock *nsk = tcp_v4_cookie_check(sk, skb);
1680
1681 if (!nsk)
1682 goto discard;
1683 if (nsk != sk) {
1684 if (tcp_child_process(sk, nsk, skb)) {
1685 rsk = nsk;
1686 goto reset;
1687 }
1688 return 0;
1689 }
1690 } else
1691 sock_rps_save_rxhash(sk, skb);
1692
1693 if (tcp_rcv_state_process(sk, skb)) {
1694 rsk = sk;
1695 goto reset;
1696 }
1697 return 0;
1698
1699 reset:
1700 tcp_v4_send_reset(rsk, skb);
1701 discard:
1702 kfree_skb_reason(skb, reason);
1703 /* Be careful here. If this function gets more complicated and
1704 * gcc suffers from register pressure on the x86, sk (in %ebx)
1705 * might be destroyed here. This current version compiles correctly,
1706 * but you have been warned.
1707 */
1708 return 0;
1709
1710 csum_err:
1711 reason = SKB_DROP_REASON_TCP_CSUM;
1712 trace_tcp_bad_csum(skb);
1713 TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1714 TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1715 goto discard;
1716 }
1717 EXPORT_SYMBOL(tcp_v4_do_rcv);
1718
tcp_v4_early_demux(struct sk_buff * skb)1719 int tcp_v4_early_demux(struct sk_buff *skb)
1720 {
1721 struct net *net = dev_net(skb->dev);
1722 const struct iphdr *iph;
1723 const struct tcphdr *th;
1724 struct sock *sk;
1725
1726 if (skb->pkt_type != PACKET_HOST)
1727 return 0;
1728
1729 if (!pskb_may_pull(skb, skb_transport_offset(skb) + sizeof(struct tcphdr)))
1730 return 0;
1731
1732 iph = ip_hdr(skb);
1733 th = tcp_hdr(skb);
1734
1735 if (th->doff < sizeof(struct tcphdr) / 4)
1736 return 0;
1737
1738 sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
1739 iph->saddr, th->source,
1740 iph->daddr, ntohs(th->dest),
1741 skb->skb_iif, inet_sdif(skb));
1742 if (sk) {
1743 skb->sk = sk;
1744 skb->destructor = sock_edemux;
1745 if (sk_fullsock(sk)) {
1746 struct dst_entry *dst = rcu_dereference(sk->sk_rx_dst);
1747
1748 if (dst)
1749 dst = dst_check(dst, 0);
1750 if (dst &&
1751 sk->sk_rx_dst_ifindex == skb->skb_iif)
1752 skb_dst_set_noref(skb, dst);
1753 }
1754 }
1755 return 0;
1756 }
1757
tcp_add_backlog(struct sock * sk,struct sk_buff * skb,enum skb_drop_reason * reason)1758 bool tcp_add_backlog(struct sock *sk, struct sk_buff *skb,
1759 enum skb_drop_reason *reason)
1760 {
1761 u32 limit, tail_gso_size, tail_gso_segs;
1762 struct skb_shared_info *shinfo;
1763 const struct tcphdr *th;
1764 struct tcphdr *thtail;
1765 struct sk_buff *tail;
1766 unsigned int hdrlen;
1767 bool fragstolen;
1768 u32 gso_segs;
1769 u32 gso_size;
1770 int delta;
1771
1772 /* In case all data was pulled from skb frags (in __pskb_pull_tail()),
1773 * we can fix skb->truesize to its real value to avoid future drops.
1774 * This is valid because skb is not yet charged to the socket.
1775 * It has been noticed pure SACK packets were sometimes dropped
1776 * (if cooked by drivers without copybreak feature).
1777 */
1778 skb_condense(skb);
1779
1780 skb_dst_drop(skb);
1781
1782 if (unlikely(tcp_checksum_complete(skb))) {
1783 bh_unlock_sock(sk);
1784 trace_tcp_bad_csum(skb);
1785 *reason = SKB_DROP_REASON_TCP_CSUM;
1786 __TCP_INC_STATS(sock_net(sk), TCP_MIB_CSUMERRORS);
1787 __TCP_INC_STATS(sock_net(sk), TCP_MIB_INERRS);
1788 return true;
1789 }
1790
1791 /* Attempt coalescing to last skb in backlog, even if we are
1792 * above the limits.
1793 * This is okay because skb capacity is limited to MAX_SKB_FRAGS.
1794 */
1795 th = (const struct tcphdr *)skb->data;
1796 hdrlen = th->doff * 4;
1797
1798 tail = sk->sk_backlog.tail;
1799 if (!tail)
1800 goto no_coalesce;
1801 thtail = (struct tcphdr *)tail->data;
1802
1803 if (TCP_SKB_CB(tail)->end_seq != TCP_SKB_CB(skb)->seq ||
1804 TCP_SKB_CB(tail)->ip_dsfield != TCP_SKB_CB(skb)->ip_dsfield ||
1805 ((TCP_SKB_CB(tail)->tcp_flags |
1806 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_SYN | TCPHDR_RST | TCPHDR_URG)) ||
1807 !((TCP_SKB_CB(tail)->tcp_flags &
1808 TCP_SKB_CB(skb)->tcp_flags) & TCPHDR_ACK) ||
1809 ((TCP_SKB_CB(tail)->tcp_flags ^
1810 TCP_SKB_CB(skb)->tcp_flags) & (TCPHDR_ECE | TCPHDR_CWR)) ||
1811 #ifdef CONFIG_TLS_DEVICE
1812 tail->decrypted != skb->decrypted ||
1813 #endif
1814 thtail->doff != th->doff ||
1815 memcmp(thtail + 1, th + 1, hdrlen - sizeof(*th)))
1816 goto no_coalesce;
1817
1818 __skb_pull(skb, hdrlen);
1819
1820 shinfo = skb_shinfo(skb);
1821 gso_size = shinfo->gso_size ?: skb->len;
1822 gso_segs = shinfo->gso_segs ?: 1;
1823
1824 shinfo = skb_shinfo(tail);
1825 tail_gso_size = shinfo->gso_size ?: (tail->len - hdrlen);
1826 tail_gso_segs = shinfo->gso_segs ?: 1;
1827
1828 if (skb_try_coalesce(tail, skb, &fragstolen, &delta)) {
1829 TCP_SKB_CB(tail)->end_seq = TCP_SKB_CB(skb)->end_seq;
1830
1831 if (likely(!before(TCP_SKB_CB(skb)->ack_seq, TCP_SKB_CB(tail)->ack_seq))) {
1832 TCP_SKB_CB(tail)->ack_seq = TCP_SKB_CB(skb)->ack_seq;
1833 thtail->window = th->window;
1834 }
1835
1836 /* We have to update both TCP_SKB_CB(tail)->tcp_flags and
1837 * thtail->fin, so that the fast path in tcp_rcv_established()
1838 * is not entered if we append a packet with a FIN.
1839 * SYN, RST, URG are not present.
1840 * ACK is set on both packets.
1841 * PSH : we do not really care in TCP stack,
1842 * at least for 'GRO' packets.
1843 */
1844 thtail->fin |= th->fin;
1845 TCP_SKB_CB(tail)->tcp_flags |= TCP_SKB_CB(skb)->tcp_flags;
1846
1847 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1848 TCP_SKB_CB(tail)->has_rxtstamp = true;
1849 tail->tstamp = skb->tstamp;
1850 skb_hwtstamps(tail)->hwtstamp = skb_hwtstamps(skb)->hwtstamp;
1851 }
1852
1853 /* Not as strict as GRO. We only need to carry mss max value */
1854 shinfo->gso_size = max(gso_size, tail_gso_size);
1855 shinfo->gso_segs = min_t(u32, gso_segs + tail_gso_segs, 0xFFFF);
1856
1857 sk->sk_backlog.len += delta;
1858 __NET_INC_STATS(sock_net(sk),
1859 LINUX_MIB_TCPBACKLOGCOALESCE);
1860 kfree_skb_partial(skb, fragstolen);
1861 return false;
1862 }
1863 __skb_push(skb, hdrlen);
1864
1865 no_coalesce:
1866 limit = (u32)READ_ONCE(sk->sk_rcvbuf) + (u32)(READ_ONCE(sk->sk_sndbuf) >> 1);
1867
1868 /* Only socket owner can try to collapse/prune rx queues
1869 * to reduce memory overhead, so add a little headroom here.
1870 * Few sockets backlog are possibly concurrently non empty.
1871 */
1872 limit += 64 * 1024;
1873
1874 if (unlikely(sk_add_backlog(sk, skb, limit))) {
1875 bh_unlock_sock(sk);
1876 *reason = SKB_DROP_REASON_SOCKET_BACKLOG;
1877 __NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPBACKLOGDROP);
1878 return true;
1879 }
1880 return false;
1881 }
1882 EXPORT_SYMBOL(tcp_add_backlog);
1883
tcp_filter(struct sock * sk,struct sk_buff * skb)1884 int tcp_filter(struct sock *sk, struct sk_buff *skb)
1885 {
1886 struct tcphdr *th = (struct tcphdr *)skb->data;
1887
1888 return sk_filter_trim_cap(sk, skb, th->doff * 4);
1889 }
1890 EXPORT_SYMBOL(tcp_filter);
1891
tcp_v4_restore_cb(struct sk_buff * skb)1892 static void tcp_v4_restore_cb(struct sk_buff *skb)
1893 {
1894 memmove(IPCB(skb), &TCP_SKB_CB(skb)->header.h4,
1895 sizeof(struct inet_skb_parm));
1896 }
1897
tcp_v4_fill_cb(struct sk_buff * skb,const struct iphdr * iph,const struct tcphdr * th)1898 static void tcp_v4_fill_cb(struct sk_buff *skb, const struct iphdr *iph,
1899 const struct tcphdr *th)
1900 {
1901 /* This is tricky : We move IPCB at its correct location into TCP_SKB_CB()
1902 * barrier() makes sure compiler wont play fool^Waliasing games.
1903 */
1904 memmove(&TCP_SKB_CB(skb)->header.h4, IPCB(skb),
1905 sizeof(struct inet_skb_parm));
1906 barrier();
1907
1908 TCP_SKB_CB(skb)->seq = ntohl(th->seq);
1909 TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
1910 skb->len - th->doff * 4);
1911 TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
1912 TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
1913 TCP_SKB_CB(skb)->tcp_tw_isn = 0;
1914 TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
1915 TCP_SKB_CB(skb)->sacked = 0;
1916 TCP_SKB_CB(skb)->has_rxtstamp =
1917 skb->tstamp || skb_hwtstamps(skb)->hwtstamp;
1918 }
1919
1920 /*
1921 * From tcp_input.c
1922 */
1923
tcp_v4_rcv(struct sk_buff * skb)1924 int tcp_v4_rcv(struct sk_buff *skb)
1925 {
1926 struct net *net = dev_net(skb->dev);
1927 enum skb_drop_reason drop_reason;
1928 int sdif = inet_sdif(skb);
1929 int dif = inet_iif(skb);
1930 const struct iphdr *iph;
1931 const struct tcphdr *th;
1932 bool refcounted;
1933 struct sock *sk;
1934 int ret;
1935
1936 drop_reason = SKB_DROP_REASON_NOT_SPECIFIED;
1937 if (skb->pkt_type != PACKET_HOST)
1938 goto discard_it;
1939
1940 /* Count it even if it's bad */
1941 __TCP_INC_STATS(net, TCP_MIB_INSEGS);
1942
1943 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
1944 goto discard_it;
1945
1946 th = (const struct tcphdr *)skb->data;
1947
1948 if (unlikely(th->doff < sizeof(struct tcphdr) / 4)) {
1949 drop_reason = SKB_DROP_REASON_PKT_TOO_SMALL;
1950 goto bad_packet;
1951 }
1952 if (!pskb_may_pull(skb, th->doff * 4))
1953 goto discard_it;
1954
1955 /* An explanation is required here, I think.
1956 * Packet length and doff are validated by header prediction,
1957 * provided case of th->doff==0 is eliminated.
1958 * So, we defer the checks. */
1959
1960 if (skb_checksum_init(skb, IPPROTO_TCP, inet_compute_pseudo))
1961 goto csum_error;
1962
1963 th = (const struct tcphdr *)skb->data;
1964 iph = ip_hdr(skb);
1965 lookup:
1966 sk = __inet_lookup_skb(net->ipv4.tcp_death_row.hashinfo,
1967 skb, __tcp_hdrlen(th), th->source,
1968 th->dest, sdif, &refcounted);
1969 if (!sk)
1970 goto no_tcp_socket;
1971
1972 process:
1973 if (sk->sk_state == TCP_TIME_WAIT)
1974 goto do_time_wait;
1975
1976 if (sk->sk_state == TCP_NEW_SYN_RECV) {
1977 struct request_sock *req = inet_reqsk(sk);
1978 bool req_stolen = false;
1979 struct sock *nsk;
1980
1981 sk = req->rsk_listener;
1982 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb))
1983 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
1984 else
1985 drop_reason = tcp_inbound_md5_hash(sk, skb,
1986 &iph->saddr, &iph->daddr,
1987 AF_INET, dif, sdif);
1988 if (unlikely(drop_reason)) {
1989 sk_drops_add(sk, skb);
1990 reqsk_put(req);
1991 goto discard_it;
1992 }
1993 if (tcp_checksum_complete(skb)) {
1994 reqsk_put(req);
1995 goto csum_error;
1996 }
1997 if (unlikely(sk->sk_state != TCP_LISTEN)) {
1998 nsk = reuseport_migrate_sock(sk, req_to_sk(req), skb);
1999 if (!nsk) {
2000 inet_csk_reqsk_queue_drop_and_put(sk, req);
2001 goto lookup;
2002 }
2003 sk = nsk;
2004 /* reuseport_migrate_sock() has already held one sk_refcnt
2005 * before returning.
2006 */
2007 } else {
2008 /* We own a reference on the listener, increase it again
2009 * as we might lose it too soon.
2010 */
2011 sock_hold(sk);
2012 }
2013 refcounted = true;
2014 nsk = NULL;
2015 if (!tcp_filter(sk, skb)) {
2016 th = (const struct tcphdr *)skb->data;
2017 iph = ip_hdr(skb);
2018 tcp_v4_fill_cb(skb, iph, th);
2019 nsk = tcp_check_req(sk, skb, req, false, &req_stolen);
2020 } else {
2021 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2022 }
2023 if (!nsk) {
2024 reqsk_put(req);
2025 if (req_stolen) {
2026 /* Another cpu got exclusive access to req
2027 * and created a full blown socket.
2028 * Try to feed this packet to this socket
2029 * instead of discarding it.
2030 */
2031 tcp_v4_restore_cb(skb);
2032 sock_put(sk);
2033 goto lookup;
2034 }
2035 goto discard_and_relse;
2036 }
2037 nf_reset_ct(skb);
2038 if (nsk == sk) {
2039 reqsk_put(req);
2040 tcp_v4_restore_cb(skb);
2041 } else if (tcp_child_process(sk, nsk, skb)) {
2042 tcp_v4_send_reset(nsk, skb);
2043 goto discard_and_relse;
2044 } else {
2045 sock_put(sk);
2046 return 0;
2047 }
2048 }
2049
2050 if (static_branch_unlikely(&ip4_min_ttl)) {
2051 /* min_ttl can be changed concurrently from do_ip_setsockopt() */
2052 if (unlikely(iph->ttl < READ_ONCE(inet_sk(sk)->min_ttl))) {
2053 __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
2054 goto discard_and_relse;
2055 }
2056 }
2057
2058 if (!xfrm4_policy_check(sk, XFRM_POLICY_IN, skb)) {
2059 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2060 goto discard_and_relse;
2061 }
2062
2063 drop_reason = tcp_inbound_md5_hash(sk, skb, &iph->saddr,
2064 &iph->daddr, AF_INET, dif, sdif);
2065 if (drop_reason)
2066 goto discard_and_relse;
2067
2068 nf_reset_ct(skb);
2069
2070 if (tcp_filter(sk, skb)) {
2071 drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
2072 goto discard_and_relse;
2073 }
2074 th = (const struct tcphdr *)skb->data;
2075 iph = ip_hdr(skb);
2076 tcp_v4_fill_cb(skb, iph, th);
2077
2078 skb->dev = NULL;
2079
2080 if (sk->sk_state == TCP_LISTEN) {
2081 ret = tcp_v4_do_rcv(sk, skb);
2082 goto put_and_return;
2083 }
2084
2085 sk_incoming_cpu_update(sk);
2086
2087 bh_lock_sock_nested(sk);
2088 tcp_segs_in(tcp_sk(sk), skb);
2089 ret = 0;
2090 if (!sock_owned_by_user(sk)) {
2091 ret = tcp_v4_do_rcv(sk, skb);
2092 } else {
2093 if (tcp_add_backlog(sk, skb, &drop_reason))
2094 goto discard_and_relse;
2095 }
2096 bh_unlock_sock(sk);
2097
2098 put_and_return:
2099 if (refcounted)
2100 sock_put(sk);
2101
2102 return ret;
2103
2104 no_tcp_socket:
2105 drop_reason = SKB_DROP_REASON_NO_SOCKET;
2106 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb))
2107 goto discard_it;
2108
2109 tcp_v4_fill_cb(skb, iph, th);
2110
2111 if (tcp_checksum_complete(skb)) {
2112 csum_error:
2113 drop_reason = SKB_DROP_REASON_TCP_CSUM;
2114 trace_tcp_bad_csum(skb);
2115 __TCP_INC_STATS(net, TCP_MIB_CSUMERRORS);
2116 bad_packet:
2117 __TCP_INC_STATS(net, TCP_MIB_INERRS);
2118 } else {
2119 tcp_v4_send_reset(NULL, skb);
2120 }
2121
2122 discard_it:
2123 SKB_DR_OR(drop_reason, NOT_SPECIFIED);
2124 /* Discard frame. */
2125 kfree_skb_reason(skb, drop_reason);
2126 return 0;
2127
2128 discard_and_relse:
2129 sk_drops_add(sk, skb);
2130 if (refcounted)
2131 sock_put(sk);
2132 goto discard_it;
2133
2134 do_time_wait:
2135 if (!xfrm4_policy_check(NULL, XFRM_POLICY_IN, skb)) {
2136 drop_reason = SKB_DROP_REASON_XFRM_POLICY;
2137 inet_twsk_put(inet_twsk(sk));
2138 goto discard_it;
2139 }
2140
2141 tcp_v4_fill_cb(skb, iph, th);
2142
2143 if (tcp_checksum_complete(skb)) {
2144 inet_twsk_put(inet_twsk(sk));
2145 goto csum_error;
2146 }
2147 switch (tcp_timewait_state_process(inet_twsk(sk), skb, th)) {
2148 case TCP_TW_SYN: {
2149 struct sock *sk2 = inet_lookup_listener(net,
2150 net->ipv4.tcp_death_row.hashinfo,
2151 skb, __tcp_hdrlen(th),
2152 iph->saddr, th->source,
2153 iph->daddr, th->dest,
2154 inet_iif(skb),
2155 sdif);
2156 if (sk2) {
2157 inet_twsk_deschedule_put(inet_twsk(sk));
2158 sk = sk2;
2159 tcp_v4_restore_cb(skb);
2160 refcounted = false;
2161 goto process;
2162 }
2163 }
2164 /* to ACK */
2165 fallthrough;
2166 case TCP_TW_ACK:
2167 tcp_v4_timewait_ack(sk, skb);
2168 break;
2169 case TCP_TW_RST:
2170 tcp_v4_send_reset(sk, skb);
2171 inet_twsk_deschedule_put(inet_twsk(sk));
2172 goto discard_it;
2173 case TCP_TW_SUCCESS:;
2174 }
2175 goto discard_it;
2176 }
2177
2178 static struct timewait_sock_ops tcp_timewait_sock_ops = {
2179 .twsk_obj_size = sizeof(struct tcp_timewait_sock),
2180 .twsk_unique = tcp_twsk_unique,
2181 .twsk_destructor= tcp_twsk_destructor,
2182 };
2183
inet_sk_rx_dst_set(struct sock * sk,const struct sk_buff * skb)2184 void inet_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
2185 {
2186 struct dst_entry *dst = skb_dst(skb);
2187
2188 if (dst && dst_hold_safe(dst)) {
2189 rcu_assign_pointer(sk->sk_rx_dst, dst);
2190 sk->sk_rx_dst_ifindex = skb->skb_iif;
2191 }
2192 }
2193 EXPORT_SYMBOL(inet_sk_rx_dst_set);
2194
2195 const struct inet_connection_sock_af_ops ipv4_specific = {
2196 .queue_xmit = ip_queue_xmit,
2197 .send_check = tcp_v4_send_check,
2198 .rebuild_header = inet_sk_rebuild_header,
2199 .sk_rx_dst_set = inet_sk_rx_dst_set,
2200 .conn_request = tcp_v4_conn_request,
2201 .syn_recv_sock = tcp_v4_syn_recv_sock,
2202 .net_header_len = sizeof(struct iphdr),
2203 .setsockopt = ip_setsockopt,
2204 .getsockopt = ip_getsockopt,
2205 .addr2sockaddr = inet_csk_addr2sockaddr,
2206 .sockaddr_len = sizeof(struct sockaddr_in),
2207 .mtu_reduced = tcp_v4_mtu_reduced,
2208 };
2209 EXPORT_SYMBOL(ipv4_specific);
2210
2211 #ifdef CONFIG_TCP_MD5SIG
2212 static const struct tcp_sock_af_ops tcp_sock_ipv4_specific = {
2213 .md5_lookup = tcp_v4_md5_lookup,
2214 .calc_md5_hash = tcp_v4_md5_hash_skb,
2215 .md5_parse = tcp_v4_parse_md5_keys,
2216 };
2217 #endif
2218
2219 /* NOTE: A lot of things set to zero explicitly by call to
2220 * sk_alloc() so need not be done here.
2221 */
tcp_v4_init_sock(struct sock * sk)2222 static int tcp_v4_init_sock(struct sock *sk)
2223 {
2224 struct inet_connection_sock *icsk = inet_csk(sk);
2225
2226 tcp_init_sock(sk);
2227
2228 icsk->icsk_af_ops = &ipv4_specific;
2229
2230 #ifdef CONFIG_TCP_MD5SIG
2231 tcp_sk(sk)->af_specific = &tcp_sock_ipv4_specific;
2232 #endif
2233
2234 return 0;
2235 }
2236
tcp_v4_destroy_sock(struct sock * sk)2237 void tcp_v4_destroy_sock(struct sock *sk)
2238 {
2239 struct tcp_sock *tp = tcp_sk(sk);
2240
2241 trace_tcp_destroy_sock(sk);
2242
2243 tcp_clear_xmit_timers(sk);
2244
2245 tcp_cleanup_congestion_control(sk);
2246
2247 tcp_cleanup_ulp(sk);
2248
2249 /* Cleanup up the write buffer. */
2250 tcp_write_queue_purge(sk);
2251
2252 /* Check if we want to disable active TFO */
2253 tcp_fastopen_active_disable_ofo_check(sk);
2254
2255 /* Cleans up our, hopefully empty, out_of_order_queue. */
2256 skb_rbtree_purge(&tp->out_of_order_queue);
2257
2258 #ifdef CONFIG_TCP_MD5SIG
2259 /* Clean up the MD5 key list, if any */
2260 if (tp->md5sig_info) {
2261 tcp_clear_md5_list(sk);
2262 kfree_rcu(rcu_dereference_protected(tp->md5sig_info, 1), rcu);
2263 tp->md5sig_info = NULL;
2264 }
2265 #endif
2266
2267 /* Clean up a referenced TCP bind bucket. */
2268 if (inet_csk(sk)->icsk_bind_hash)
2269 inet_put_port(sk);
2270
2271 BUG_ON(rcu_access_pointer(tp->fastopen_rsk));
2272
2273 /* If socket is aborted during connect operation */
2274 tcp_free_fastopen_req(tp);
2275 tcp_fastopen_destroy_cipher(sk);
2276 tcp_saved_syn_free(tp);
2277
2278 sk_sockets_allocated_dec(sk);
2279 }
2280 EXPORT_SYMBOL(tcp_v4_destroy_sock);
2281
2282 #ifdef CONFIG_PROC_FS
2283 /* Proc filesystem TCP sock list dumping. */
2284
2285 static unsigned short seq_file_family(const struct seq_file *seq);
2286
seq_sk_match(struct seq_file * seq,const struct sock * sk)2287 static bool seq_sk_match(struct seq_file *seq, const struct sock *sk)
2288 {
2289 unsigned short family = seq_file_family(seq);
2290
2291 /* AF_UNSPEC is used as a match all */
2292 return ((family == AF_UNSPEC || family == sk->sk_family) &&
2293 net_eq(sock_net(sk), seq_file_net(seq)));
2294 }
2295
2296 /* Find a non empty bucket (starting from st->bucket)
2297 * and return the first sk from it.
2298 */
listening_get_first(struct seq_file * seq)2299 static void *listening_get_first(struct seq_file *seq)
2300 {
2301 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2302 struct tcp_iter_state *st = seq->private;
2303
2304 st->offset = 0;
2305 for (; st->bucket <= hinfo->lhash2_mask; st->bucket++) {
2306 struct inet_listen_hashbucket *ilb2;
2307 struct hlist_nulls_node *node;
2308 struct sock *sk;
2309
2310 ilb2 = &hinfo->lhash2[st->bucket];
2311 if (hlist_nulls_empty(&ilb2->nulls_head))
2312 continue;
2313
2314 spin_lock(&ilb2->lock);
2315 sk_nulls_for_each(sk, node, &ilb2->nulls_head) {
2316 if (seq_sk_match(seq, sk))
2317 return sk;
2318 }
2319 spin_unlock(&ilb2->lock);
2320 }
2321
2322 return NULL;
2323 }
2324
2325 /* Find the next sk of "cur" within the same bucket (i.e. st->bucket).
2326 * If "cur" is the last one in the st->bucket,
2327 * call listening_get_first() to return the first sk of the next
2328 * non empty bucket.
2329 */
listening_get_next(struct seq_file * seq,void * cur)2330 static void *listening_get_next(struct seq_file *seq, void *cur)
2331 {
2332 struct tcp_iter_state *st = seq->private;
2333 struct inet_listen_hashbucket *ilb2;
2334 struct hlist_nulls_node *node;
2335 struct inet_hashinfo *hinfo;
2336 struct sock *sk = cur;
2337
2338 ++st->num;
2339 ++st->offset;
2340
2341 sk = sk_nulls_next(sk);
2342 sk_nulls_for_each_from(sk, node) {
2343 if (seq_sk_match(seq, sk))
2344 return sk;
2345 }
2346
2347 hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2348 ilb2 = &hinfo->lhash2[st->bucket];
2349 spin_unlock(&ilb2->lock);
2350 ++st->bucket;
2351 return listening_get_first(seq);
2352 }
2353
listening_get_idx(struct seq_file * seq,loff_t * pos)2354 static void *listening_get_idx(struct seq_file *seq, loff_t *pos)
2355 {
2356 struct tcp_iter_state *st = seq->private;
2357 void *rc;
2358
2359 st->bucket = 0;
2360 st->offset = 0;
2361 rc = listening_get_first(seq);
2362
2363 while (rc && *pos) {
2364 rc = listening_get_next(seq, rc);
2365 --*pos;
2366 }
2367 return rc;
2368 }
2369
empty_bucket(struct inet_hashinfo * hinfo,const struct tcp_iter_state * st)2370 static inline bool empty_bucket(struct inet_hashinfo *hinfo,
2371 const struct tcp_iter_state *st)
2372 {
2373 return hlist_nulls_empty(&hinfo->ehash[st->bucket].chain);
2374 }
2375
2376 /*
2377 * Get first established socket starting from bucket given in st->bucket.
2378 * If st->bucket is zero, the very first socket in the hash is returned.
2379 */
established_get_first(struct seq_file * seq)2380 static void *established_get_first(struct seq_file *seq)
2381 {
2382 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2383 struct tcp_iter_state *st = seq->private;
2384
2385 st->offset = 0;
2386 for (; st->bucket <= hinfo->ehash_mask; ++st->bucket) {
2387 struct sock *sk;
2388 struct hlist_nulls_node *node;
2389 spinlock_t *lock = inet_ehash_lockp(hinfo, st->bucket);
2390
2391 /* Lockless fast path for the common case of empty buckets */
2392 if (empty_bucket(hinfo, st))
2393 continue;
2394
2395 spin_lock_bh(lock);
2396 sk_nulls_for_each(sk, node, &hinfo->ehash[st->bucket].chain) {
2397 if (seq_sk_match(seq, sk))
2398 return sk;
2399 }
2400 spin_unlock_bh(lock);
2401 }
2402
2403 return NULL;
2404 }
2405
established_get_next(struct seq_file * seq,void * cur)2406 static void *established_get_next(struct seq_file *seq, void *cur)
2407 {
2408 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2409 struct tcp_iter_state *st = seq->private;
2410 struct hlist_nulls_node *node;
2411 struct sock *sk = cur;
2412
2413 ++st->num;
2414 ++st->offset;
2415
2416 sk = sk_nulls_next(sk);
2417
2418 sk_nulls_for_each_from(sk, node) {
2419 if (seq_sk_match(seq, sk))
2420 return sk;
2421 }
2422
2423 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2424 ++st->bucket;
2425 return established_get_first(seq);
2426 }
2427
established_get_idx(struct seq_file * seq,loff_t pos)2428 static void *established_get_idx(struct seq_file *seq, loff_t pos)
2429 {
2430 struct tcp_iter_state *st = seq->private;
2431 void *rc;
2432
2433 st->bucket = 0;
2434 rc = established_get_first(seq);
2435
2436 while (rc && pos) {
2437 rc = established_get_next(seq, rc);
2438 --pos;
2439 }
2440 return rc;
2441 }
2442
tcp_get_idx(struct seq_file * seq,loff_t pos)2443 static void *tcp_get_idx(struct seq_file *seq, loff_t pos)
2444 {
2445 void *rc;
2446 struct tcp_iter_state *st = seq->private;
2447
2448 st->state = TCP_SEQ_STATE_LISTENING;
2449 rc = listening_get_idx(seq, &pos);
2450
2451 if (!rc) {
2452 st->state = TCP_SEQ_STATE_ESTABLISHED;
2453 rc = established_get_idx(seq, pos);
2454 }
2455
2456 return rc;
2457 }
2458
tcp_seek_last_pos(struct seq_file * seq)2459 static void *tcp_seek_last_pos(struct seq_file *seq)
2460 {
2461 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2462 struct tcp_iter_state *st = seq->private;
2463 int bucket = st->bucket;
2464 int offset = st->offset;
2465 int orig_num = st->num;
2466 void *rc = NULL;
2467
2468 switch (st->state) {
2469 case TCP_SEQ_STATE_LISTENING:
2470 if (st->bucket > hinfo->lhash2_mask)
2471 break;
2472 st->state = TCP_SEQ_STATE_LISTENING;
2473 rc = listening_get_first(seq);
2474 while (offset-- && rc && bucket == st->bucket)
2475 rc = listening_get_next(seq, rc);
2476 if (rc)
2477 break;
2478 st->bucket = 0;
2479 st->state = TCP_SEQ_STATE_ESTABLISHED;
2480 fallthrough;
2481 case TCP_SEQ_STATE_ESTABLISHED:
2482 if (st->bucket > hinfo->ehash_mask)
2483 break;
2484 rc = established_get_first(seq);
2485 while (offset-- && rc && bucket == st->bucket)
2486 rc = established_get_next(seq, rc);
2487 }
2488
2489 st->num = orig_num;
2490
2491 return rc;
2492 }
2493
tcp_seq_start(struct seq_file * seq,loff_t * pos)2494 void *tcp_seq_start(struct seq_file *seq, loff_t *pos)
2495 {
2496 struct tcp_iter_state *st = seq->private;
2497 void *rc;
2498
2499 if (*pos && *pos == st->last_pos) {
2500 rc = tcp_seek_last_pos(seq);
2501 if (rc)
2502 goto out;
2503 }
2504
2505 st->state = TCP_SEQ_STATE_LISTENING;
2506 st->num = 0;
2507 st->bucket = 0;
2508 st->offset = 0;
2509 rc = *pos ? tcp_get_idx(seq, *pos - 1) : SEQ_START_TOKEN;
2510
2511 out:
2512 st->last_pos = *pos;
2513 return rc;
2514 }
2515 EXPORT_SYMBOL(tcp_seq_start);
2516
tcp_seq_next(struct seq_file * seq,void * v,loff_t * pos)2517 void *tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2518 {
2519 struct tcp_iter_state *st = seq->private;
2520 void *rc = NULL;
2521
2522 if (v == SEQ_START_TOKEN) {
2523 rc = tcp_get_idx(seq, 0);
2524 goto out;
2525 }
2526
2527 switch (st->state) {
2528 case TCP_SEQ_STATE_LISTENING:
2529 rc = listening_get_next(seq, v);
2530 if (!rc) {
2531 st->state = TCP_SEQ_STATE_ESTABLISHED;
2532 st->bucket = 0;
2533 st->offset = 0;
2534 rc = established_get_first(seq);
2535 }
2536 break;
2537 case TCP_SEQ_STATE_ESTABLISHED:
2538 rc = established_get_next(seq, v);
2539 break;
2540 }
2541 out:
2542 ++*pos;
2543 st->last_pos = *pos;
2544 return rc;
2545 }
2546 EXPORT_SYMBOL(tcp_seq_next);
2547
tcp_seq_stop(struct seq_file * seq,void * v)2548 void tcp_seq_stop(struct seq_file *seq, void *v)
2549 {
2550 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2551 struct tcp_iter_state *st = seq->private;
2552
2553 switch (st->state) {
2554 case TCP_SEQ_STATE_LISTENING:
2555 if (v != SEQ_START_TOKEN)
2556 spin_unlock(&hinfo->lhash2[st->bucket].lock);
2557 break;
2558 case TCP_SEQ_STATE_ESTABLISHED:
2559 if (v)
2560 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2561 break;
2562 }
2563 }
2564 EXPORT_SYMBOL(tcp_seq_stop);
2565
get_openreq4(const struct request_sock * req,struct seq_file * f,int i)2566 static void get_openreq4(const struct request_sock *req,
2567 struct seq_file *f, int i)
2568 {
2569 const struct inet_request_sock *ireq = inet_rsk(req);
2570 long delta = req->rsk_timer.expires - jiffies;
2571
2572 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2573 " %02X %08X:%08X %02X:%08lX %08X %5u %8d %u %d %pK",
2574 i,
2575 ireq->ir_loc_addr,
2576 ireq->ir_num,
2577 ireq->ir_rmt_addr,
2578 ntohs(ireq->ir_rmt_port),
2579 TCP_SYN_RECV,
2580 0, 0, /* could print option size, but that is af dependent. */
2581 1, /* timers active (only the expire timer) */
2582 jiffies_delta_to_clock_t(delta),
2583 req->num_timeout,
2584 from_kuid_munged(seq_user_ns(f),
2585 sock_i_uid(req->rsk_listener)),
2586 0, /* non standard timer */
2587 0, /* open_requests have no inode */
2588 0,
2589 req);
2590 }
2591
get_tcp4_sock(struct sock * sk,struct seq_file * f,int i)2592 static void get_tcp4_sock(struct sock *sk, struct seq_file *f, int i)
2593 {
2594 int timer_active;
2595 unsigned long timer_expires;
2596 const struct tcp_sock *tp = tcp_sk(sk);
2597 const struct inet_connection_sock *icsk = inet_csk(sk);
2598 const struct inet_sock *inet = inet_sk(sk);
2599 const struct fastopen_queue *fastopenq = &icsk->icsk_accept_queue.fastopenq;
2600 __be32 dest = inet->inet_daddr;
2601 __be32 src = inet->inet_rcv_saddr;
2602 __u16 destp = ntohs(inet->inet_dport);
2603 __u16 srcp = ntohs(inet->inet_sport);
2604 int rx_queue;
2605 int state;
2606
2607 if (icsk->icsk_pending == ICSK_TIME_RETRANS ||
2608 icsk->icsk_pending == ICSK_TIME_REO_TIMEOUT ||
2609 icsk->icsk_pending == ICSK_TIME_LOSS_PROBE) {
2610 timer_active = 1;
2611 timer_expires = icsk->icsk_timeout;
2612 } else if (icsk->icsk_pending == ICSK_TIME_PROBE0) {
2613 timer_active = 4;
2614 timer_expires = icsk->icsk_timeout;
2615 } else if (timer_pending(&sk->sk_timer)) {
2616 timer_active = 2;
2617 timer_expires = sk->sk_timer.expires;
2618 } else {
2619 timer_active = 0;
2620 timer_expires = jiffies;
2621 }
2622
2623 state = inet_sk_state_load(sk);
2624 if (state == TCP_LISTEN)
2625 rx_queue = READ_ONCE(sk->sk_ack_backlog);
2626 else
2627 /* Because we don't lock the socket,
2628 * we might find a transient negative value.
2629 */
2630 rx_queue = max_t(int, READ_ONCE(tp->rcv_nxt) -
2631 READ_ONCE(tp->copied_seq), 0);
2632
2633 seq_printf(f, "%4d: %08X:%04X %08X:%04X %02X %08X:%08X %02X:%08lX "
2634 "%08X %5u %8d %lu %d %pK %lu %lu %u %u %d",
2635 i, src, srcp, dest, destp, state,
2636 READ_ONCE(tp->write_seq) - tp->snd_una,
2637 rx_queue,
2638 timer_active,
2639 jiffies_delta_to_clock_t(timer_expires - jiffies),
2640 icsk->icsk_retransmits,
2641 from_kuid_munged(seq_user_ns(f), sock_i_uid(sk)),
2642 icsk->icsk_probes_out,
2643 sock_i_ino(sk),
2644 refcount_read(&sk->sk_refcnt), sk,
2645 jiffies_to_clock_t(icsk->icsk_rto),
2646 jiffies_to_clock_t(icsk->icsk_ack.ato),
2647 (icsk->icsk_ack.quick << 1) | inet_csk_in_pingpong_mode(sk),
2648 tcp_snd_cwnd(tp),
2649 state == TCP_LISTEN ?
2650 fastopenq->max_qlen :
2651 (tcp_in_initial_slowstart(tp) ? -1 : tp->snd_ssthresh));
2652 }
2653
get_timewait4_sock(const struct inet_timewait_sock * tw,struct seq_file * f,int i)2654 static void get_timewait4_sock(const struct inet_timewait_sock *tw,
2655 struct seq_file *f, int i)
2656 {
2657 long delta = tw->tw_timer.expires - jiffies;
2658 __be32 dest, src;
2659 __u16 destp, srcp;
2660
2661 dest = tw->tw_daddr;
2662 src = tw->tw_rcv_saddr;
2663 destp = ntohs(tw->tw_dport);
2664 srcp = ntohs(tw->tw_sport);
2665
2666 seq_printf(f, "%4d: %08X:%04X %08X:%04X"
2667 " %02X %08X:%08X %02X:%08lX %08X %5d %8d %d %d %pK",
2668 i, src, srcp, dest, destp, tw->tw_substate, 0, 0,
2669 3, jiffies_delta_to_clock_t(delta), 0, 0, 0, 0,
2670 refcount_read(&tw->tw_refcnt), tw);
2671 }
2672
2673 #define TMPSZ 150
2674
tcp4_seq_show(struct seq_file * seq,void * v)2675 static int tcp4_seq_show(struct seq_file *seq, void *v)
2676 {
2677 struct tcp_iter_state *st;
2678 struct sock *sk = v;
2679
2680 seq_setwidth(seq, TMPSZ - 1);
2681 if (v == SEQ_START_TOKEN) {
2682 seq_puts(seq, " sl local_address rem_address st tx_queue "
2683 "rx_queue tr tm->when retrnsmt uid timeout "
2684 "inode");
2685 goto out;
2686 }
2687 st = seq->private;
2688
2689 if (sk->sk_state == TCP_TIME_WAIT)
2690 get_timewait4_sock(v, seq, st->num);
2691 else if (sk->sk_state == TCP_NEW_SYN_RECV)
2692 get_openreq4(v, seq, st->num);
2693 else
2694 get_tcp4_sock(v, seq, st->num);
2695 out:
2696 seq_pad(seq, '\n');
2697 return 0;
2698 }
2699
2700 #ifdef CONFIG_BPF_SYSCALL
2701 struct bpf_tcp_iter_state {
2702 struct tcp_iter_state state;
2703 unsigned int cur_sk;
2704 unsigned int end_sk;
2705 unsigned int max_sk;
2706 struct sock **batch;
2707 bool st_bucket_done;
2708 };
2709
2710 struct bpf_iter__tcp {
2711 __bpf_md_ptr(struct bpf_iter_meta *, meta);
2712 __bpf_md_ptr(struct sock_common *, sk_common);
2713 uid_t uid __aligned(8);
2714 };
2715
tcp_prog_seq_show(struct bpf_prog * prog,struct bpf_iter_meta * meta,struct sock_common * sk_common,uid_t uid)2716 static int tcp_prog_seq_show(struct bpf_prog *prog, struct bpf_iter_meta *meta,
2717 struct sock_common *sk_common, uid_t uid)
2718 {
2719 struct bpf_iter__tcp ctx;
2720
2721 meta->seq_num--; /* skip SEQ_START_TOKEN */
2722 ctx.meta = meta;
2723 ctx.sk_common = sk_common;
2724 ctx.uid = uid;
2725 return bpf_iter_run_prog(prog, &ctx);
2726 }
2727
bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state * iter)2728 static void bpf_iter_tcp_put_batch(struct bpf_tcp_iter_state *iter)
2729 {
2730 while (iter->cur_sk < iter->end_sk)
2731 sock_put(iter->batch[iter->cur_sk++]);
2732 }
2733
bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state * iter,unsigned int new_batch_sz)2734 static int bpf_iter_tcp_realloc_batch(struct bpf_tcp_iter_state *iter,
2735 unsigned int new_batch_sz)
2736 {
2737 struct sock **new_batch;
2738
2739 new_batch = kvmalloc(sizeof(*new_batch) * new_batch_sz,
2740 GFP_USER | __GFP_NOWARN);
2741 if (!new_batch)
2742 return -ENOMEM;
2743
2744 bpf_iter_tcp_put_batch(iter);
2745 kvfree(iter->batch);
2746 iter->batch = new_batch;
2747 iter->max_sk = new_batch_sz;
2748
2749 return 0;
2750 }
2751
bpf_iter_tcp_listening_batch(struct seq_file * seq,struct sock * start_sk)2752 static unsigned int bpf_iter_tcp_listening_batch(struct seq_file *seq,
2753 struct sock *start_sk)
2754 {
2755 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2756 struct bpf_tcp_iter_state *iter = seq->private;
2757 struct tcp_iter_state *st = &iter->state;
2758 struct hlist_nulls_node *node;
2759 unsigned int expected = 1;
2760 struct sock *sk;
2761
2762 sock_hold(start_sk);
2763 iter->batch[iter->end_sk++] = start_sk;
2764
2765 sk = sk_nulls_next(start_sk);
2766 sk_nulls_for_each_from(sk, node) {
2767 if (seq_sk_match(seq, sk)) {
2768 if (iter->end_sk < iter->max_sk) {
2769 sock_hold(sk);
2770 iter->batch[iter->end_sk++] = sk;
2771 }
2772 expected++;
2773 }
2774 }
2775 spin_unlock(&hinfo->lhash2[st->bucket].lock);
2776
2777 return expected;
2778 }
2779
bpf_iter_tcp_established_batch(struct seq_file * seq,struct sock * start_sk)2780 static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq,
2781 struct sock *start_sk)
2782 {
2783 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2784 struct bpf_tcp_iter_state *iter = seq->private;
2785 struct tcp_iter_state *st = &iter->state;
2786 struct hlist_nulls_node *node;
2787 unsigned int expected = 1;
2788 struct sock *sk;
2789
2790 sock_hold(start_sk);
2791 iter->batch[iter->end_sk++] = start_sk;
2792
2793 sk = sk_nulls_next(start_sk);
2794 sk_nulls_for_each_from(sk, node) {
2795 if (seq_sk_match(seq, sk)) {
2796 if (iter->end_sk < iter->max_sk) {
2797 sock_hold(sk);
2798 iter->batch[iter->end_sk++] = sk;
2799 }
2800 expected++;
2801 }
2802 }
2803 spin_unlock_bh(inet_ehash_lockp(hinfo, st->bucket));
2804
2805 return expected;
2806 }
2807
bpf_iter_tcp_batch(struct seq_file * seq)2808 static struct sock *bpf_iter_tcp_batch(struct seq_file *seq)
2809 {
2810 struct inet_hashinfo *hinfo = seq_file_net(seq)->ipv4.tcp_death_row.hashinfo;
2811 struct bpf_tcp_iter_state *iter = seq->private;
2812 struct tcp_iter_state *st = &iter->state;
2813 unsigned int expected;
2814 bool resized = false;
2815 struct sock *sk;
2816
2817 /* The st->bucket is done. Directly advance to the next
2818 * bucket instead of having the tcp_seek_last_pos() to skip
2819 * one by one in the current bucket and eventually find out
2820 * it has to advance to the next bucket.
2821 */
2822 if (iter->st_bucket_done) {
2823 st->offset = 0;
2824 st->bucket++;
2825 if (st->state == TCP_SEQ_STATE_LISTENING &&
2826 st->bucket > hinfo->lhash2_mask) {
2827 st->state = TCP_SEQ_STATE_ESTABLISHED;
2828 st->bucket = 0;
2829 }
2830 }
2831
2832 again:
2833 /* Get a new batch */
2834 iter->cur_sk = 0;
2835 iter->end_sk = 0;
2836 iter->st_bucket_done = false;
2837
2838 sk = tcp_seek_last_pos(seq);
2839 if (!sk)
2840 return NULL; /* Done */
2841
2842 if (st->state == TCP_SEQ_STATE_LISTENING)
2843 expected = bpf_iter_tcp_listening_batch(seq, sk);
2844 else
2845 expected = bpf_iter_tcp_established_batch(seq, sk);
2846
2847 if (iter->end_sk == expected) {
2848 iter->st_bucket_done = true;
2849 return sk;
2850 }
2851
2852 if (!resized && !bpf_iter_tcp_realloc_batch(iter, expected * 3 / 2)) {
2853 resized = true;
2854 goto again;
2855 }
2856
2857 return sk;
2858 }
2859
bpf_iter_tcp_seq_start(struct seq_file * seq,loff_t * pos)2860 static void *bpf_iter_tcp_seq_start(struct seq_file *seq, loff_t *pos)
2861 {
2862 /* bpf iter does not support lseek, so it always
2863 * continue from where it was stop()-ped.
2864 */
2865 if (*pos)
2866 return bpf_iter_tcp_batch(seq);
2867
2868 return SEQ_START_TOKEN;
2869 }
2870
bpf_iter_tcp_seq_next(struct seq_file * seq,void * v,loff_t * pos)2871 static void *bpf_iter_tcp_seq_next(struct seq_file *seq, void *v, loff_t *pos)
2872 {
2873 struct bpf_tcp_iter_state *iter = seq->private;
2874 struct tcp_iter_state *st = &iter->state;
2875 struct sock *sk;
2876
2877 /* Whenever seq_next() is called, the iter->cur_sk is
2878 * done with seq_show(), so advance to the next sk in
2879 * the batch.
2880 */
2881 if (iter->cur_sk < iter->end_sk) {
2882 /* Keeping st->num consistent in tcp_iter_state.
2883 * bpf_iter_tcp does not use st->num.
2884 * meta.seq_num is used instead.
2885 */
2886 st->num++;
2887 /* Move st->offset to the next sk in the bucket such that
2888 * the future start() will resume at st->offset in
2889 * st->bucket. See tcp_seek_last_pos().
2890 */
2891 st->offset++;
2892 sock_put(iter->batch[iter->cur_sk++]);
2893 }
2894
2895 if (iter->cur_sk < iter->end_sk)
2896 sk = iter->batch[iter->cur_sk];
2897 else
2898 sk = bpf_iter_tcp_batch(seq);
2899
2900 ++*pos;
2901 /* Keeping st->last_pos consistent in tcp_iter_state.
2902 * bpf iter does not do lseek, so st->last_pos always equals to *pos.
2903 */
2904 st->last_pos = *pos;
2905 return sk;
2906 }
2907
bpf_iter_tcp_seq_show(struct seq_file * seq,void * v)2908 static int bpf_iter_tcp_seq_show(struct seq_file *seq, void *v)
2909 {
2910 struct bpf_iter_meta meta;
2911 struct bpf_prog *prog;
2912 struct sock *sk = v;
2913 bool slow;
2914 uid_t uid;
2915 int ret;
2916
2917 if (v == SEQ_START_TOKEN)
2918 return 0;
2919
2920 if (sk_fullsock(sk))
2921 slow = lock_sock_fast(sk);
2922
2923 if (unlikely(sk_unhashed(sk))) {
2924 ret = SEQ_SKIP;
2925 goto unlock;
2926 }
2927
2928 if (sk->sk_state == TCP_TIME_WAIT) {
2929 uid = 0;
2930 } else if (sk->sk_state == TCP_NEW_SYN_RECV) {
2931 const struct request_sock *req = v;
2932
2933 uid = from_kuid_munged(seq_user_ns(seq),
2934 sock_i_uid(req->rsk_listener));
2935 } else {
2936 uid = from_kuid_munged(seq_user_ns(seq), sock_i_uid(sk));
2937 }
2938
2939 meta.seq = seq;
2940 prog = bpf_iter_get_info(&meta, false);
2941 ret = tcp_prog_seq_show(prog, &meta, v, uid);
2942
2943 unlock:
2944 if (sk_fullsock(sk))
2945 unlock_sock_fast(sk, slow);
2946 return ret;
2947
2948 }
2949
bpf_iter_tcp_seq_stop(struct seq_file * seq,void * v)2950 static void bpf_iter_tcp_seq_stop(struct seq_file *seq, void *v)
2951 {
2952 struct bpf_tcp_iter_state *iter = seq->private;
2953 struct bpf_iter_meta meta;
2954 struct bpf_prog *prog;
2955
2956 if (!v) {
2957 meta.seq = seq;
2958 prog = bpf_iter_get_info(&meta, true);
2959 if (prog)
2960 (void)tcp_prog_seq_show(prog, &meta, v, 0);
2961 }
2962
2963 if (iter->cur_sk < iter->end_sk) {
2964 bpf_iter_tcp_put_batch(iter);
2965 iter->st_bucket_done = false;
2966 }
2967 }
2968
2969 static const struct seq_operations bpf_iter_tcp_seq_ops = {
2970 .show = bpf_iter_tcp_seq_show,
2971 .start = bpf_iter_tcp_seq_start,
2972 .next = bpf_iter_tcp_seq_next,
2973 .stop = bpf_iter_tcp_seq_stop,
2974 };
2975 #endif
seq_file_family(const struct seq_file * seq)2976 static unsigned short seq_file_family(const struct seq_file *seq)
2977 {
2978 const struct tcp_seq_afinfo *afinfo;
2979
2980 #ifdef CONFIG_BPF_SYSCALL
2981 /* Iterated from bpf_iter. Let the bpf prog to filter instead. */
2982 if (seq->op == &bpf_iter_tcp_seq_ops)
2983 return AF_UNSPEC;
2984 #endif
2985
2986 /* Iterated from proc fs */
2987 afinfo = pde_data(file_inode(seq->file));
2988 return afinfo->family;
2989 }
2990
2991 static const struct seq_operations tcp4_seq_ops = {
2992 .show = tcp4_seq_show,
2993 .start = tcp_seq_start,
2994 .next = tcp_seq_next,
2995 .stop = tcp_seq_stop,
2996 };
2997
2998 static struct tcp_seq_afinfo tcp4_seq_afinfo = {
2999 .family = AF_INET,
3000 };
3001
tcp4_proc_init_net(struct net * net)3002 static int __net_init tcp4_proc_init_net(struct net *net)
3003 {
3004 if (!proc_create_net_data("tcp", 0444, net->proc_net, &tcp4_seq_ops,
3005 sizeof(struct tcp_iter_state), &tcp4_seq_afinfo))
3006 return -ENOMEM;
3007 return 0;
3008 }
3009
tcp4_proc_exit_net(struct net * net)3010 static void __net_exit tcp4_proc_exit_net(struct net *net)
3011 {
3012 remove_proc_entry("tcp", net->proc_net);
3013 }
3014
3015 static struct pernet_operations tcp4_net_ops = {
3016 .init = tcp4_proc_init_net,
3017 .exit = tcp4_proc_exit_net,
3018 };
3019
tcp4_proc_init(void)3020 int __init tcp4_proc_init(void)
3021 {
3022 return register_pernet_subsys(&tcp4_net_ops);
3023 }
3024
tcp4_proc_exit(void)3025 void tcp4_proc_exit(void)
3026 {
3027 unregister_pernet_subsys(&tcp4_net_ops);
3028 }
3029 #endif /* CONFIG_PROC_FS */
3030
3031 /* @wake is one when sk_stream_write_space() calls us.
3032 * This sends EPOLLOUT only if notsent_bytes is half the limit.
3033 * This mimics the strategy used in sock_def_write_space().
3034 */
tcp_stream_memory_free(const struct sock * sk,int wake)3035 bool tcp_stream_memory_free(const struct sock *sk, int wake)
3036 {
3037 const struct tcp_sock *tp = tcp_sk(sk);
3038 u32 notsent_bytes = READ_ONCE(tp->write_seq) -
3039 READ_ONCE(tp->snd_nxt);
3040
3041 return (notsent_bytes << wake) < tcp_notsent_lowat(tp);
3042 }
3043 EXPORT_SYMBOL(tcp_stream_memory_free);
3044
3045 struct proto tcp_prot = {
3046 .name = "TCP",
3047 .owner = THIS_MODULE,
3048 .close = tcp_close,
3049 .pre_connect = tcp_v4_pre_connect,
3050 .connect = tcp_v4_connect,
3051 .disconnect = tcp_disconnect,
3052 .accept = inet_csk_accept,
3053 .ioctl = tcp_ioctl,
3054 .init = tcp_v4_init_sock,
3055 .destroy = tcp_v4_destroy_sock,
3056 .shutdown = tcp_shutdown,
3057 .setsockopt = tcp_setsockopt,
3058 .getsockopt = tcp_getsockopt,
3059 .bpf_bypass_getsockopt = tcp_bpf_bypass_getsockopt,
3060 .keepalive = tcp_set_keepalive,
3061 .recvmsg = tcp_recvmsg,
3062 .sendmsg = tcp_sendmsg,
3063 .sendpage = tcp_sendpage,
3064 .backlog_rcv = tcp_v4_do_rcv,
3065 .release_cb = tcp_release_cb,
3066 .hash = inet_hash,
3067 .unhash = inet_unhash,
3068 .get_port = inet_csk_get_port,
3069 .put_port = inet_put_port,
3070 #ifdef CONFIG_BPF_SYSCALL
3071 .psock_update_sk_prot = tcp_bpf_update_proto,
3072 #endif
3073 .enter_memory_pressure = tcp_enter_memory_pressure,
3074 .leave_memory_pressure = tcp_leave_memory_pressure,
3075 .stream_memory_free = tcp_stream_memory_free,
3076 .sockets_allocated = &tcp_sockets_allocated,
3077 .orphan_count = &tcp_orphan_count,
3078
3079 .memory_allocated = &tcp_memory_allocated,
3080 .per_cpu_fw_alloc = &tcp_memory_per_cpu_fw_alloc,
3081
3082 .memory_pressure = &tcp_memory_pressure,
3083 .sysctl_mem = sysctl_tcp_mem,
3084 .sysctl_wmem_offset = offsetof(struct net, ipv4.sysctl_tcp_wmem),
3085 .sysctl_rmem_offset = offsetof(struct net, ipv4.sysctl_tcp_rmem),
3086 .max_header = MAX_TCP_HEADER,
3087 .obj_size = sizeof(struct tcp_sock),
3088 .slab_flags = SLAB_TYPESAFE_BY_RCU,
3089 .twsk_prot = &tcp_timewait_sock_ops,
3090 .rsk_prot = &tcp_request_sock_ops,
3091 .h.hashinfo = NULL,
3092 .no_autobind = true,
3093 .diag_destroy = tcp_abort,
3094 };
3095 EXPORT_SYMBOL(tcp_prot);
3096
tcp_sk_exit(struct net * net)3097 static void __net_exit tcp_sk_exit(struct net *net)
3098 {
3099 if (net->ipv4.tcp_congestion_control)
3100 bpf_module_put(net->ipv4.tcp_congestion_control,
3101 net->ipv4.tcp_congestion_control->owner);
3102 }
3103
tcp_set_hashinfo(struct net * net)3104 static void __net_init tcp_set_hashinfo(struct net *net)
3105 {
3106 struct inet_hashinfo *hinfo;
3107 unsigned int ehash_entries;
3108 struct net *old_net;
3109
3110 if (net_eq(net, &init_net))
3111 goto fallback;
3112
3113 old_net = current->nsproxy->net_ns;
3114 ehash_entries = READ_ONCE(old_net->ipv4.sysctl_tcp_child_ehash_entries);
3115 if (!ehash_entries)
3116 goto fallback;
3117
3118 ehash_entries = roundup_pow_of_two(ehash_entries);
3119 hinfo = inet_pernet_hashinfo_alloc(&tcp_hashinfo, ehash_entries);
3120 if (!hinfo) {
3121 pr_warn("Failed to allocate TCP ehash (entries: %u) "
3122 "for a netns, fallback to the global one\n",
3123 ehash_entries);
3124 fallback:
3125 hinfo = &tcp_hashinfo;
3126 ehash_entries = tcp_hashinfo.ehash_mask + 1;
3127 }
3128
3129 net->ipv4.tcp_death_row.hashinfo = hinfo;
3130 net->ipv4.tcp_death_row.sysctl_max_tw_buckets = ehash_entries / 2;
3131 net->ipv4.sysctl_max_syn_backlog = max(128U, ehash_entries / 128);
3132 }
3133
tcp_sk_init(struct net * net)3134 static int __net_init tcp_sk_init(struct net *net)
3135 {
3136 net->ipv4.sysctl_tcp_ecn = 2;
3137 net->ipv4.sysctl_tcp_ecn_fallback = 1;
3138
3139 net->ipv4.sysctl_tcp_base_mss = TCP_BASE_MSS;
3140 net->ipv4.sysctl_tcp_min_snd_mss = TCP_MIN_SND_MSS;
3141 net->ipv4.sysctl_tcp_probe_threshold = TCP_PROBE_THRESHOLD;
3142 net->ipv4.sysctl_tcp_probe_interval = TCP_PROBE_INTERVAL;
3143 net->ipv4.sysctl_tcp_mtu_probe_floor = TCP_MIN_SND_MSS;
3144
3145 net->ipv4.sysctl_tcp_keepalive_time = TCP_KEEPALIVE_TIME;
3146 net->ipv4.sysctl_tcp_keepalive_probes = TCP_KEEPALIVE_PROBES;
3147 net->ipv4.sysctl_tcp_keepalive_intvl = TCP_KEEPALIVE_INTVL;
3148
3149 net->ipv4.sysctl_tcp_syn_retries = TCP_SYN_RETRIES;
3150 net->ipv4.sysctl_tcp_synack_retries = TCP_SYNACK_RETRIES;
3151 net->ipv4.sysctl_tcp_syncookies = 1;
3152 net->ipv4.sysctl_tcp_reordering = TCP_FASTRETRANS_THRESH;
3153 net->ipv4.sysctl_tcp_retries1 = TCP_RETR1;
3154 net->ipv4.sysctl_tcp_retries2 = TCP_RETR2;
3155 net->ipv4.sysctl_tcp_orphan_retries = 0;
3156 net->ipv4.sysctl_tcp_fin_timeout = TCP_FIN_TIMEOUT;
3157 net->ipv4.sysctl_tcp_notsent_lowat = UINT_MAX;
3158 net->ipv4.sysctl_tcp_tw_reuse = 2;
3159 net->ipv4.sysctl_tcp_no_ssthresh_metrics_save = 1;
3160
3161 refcount_set(&net->ipv4.tcp_death_row.tw_refcount, 1);
3162 tcp_set_hashinfo(net);
3163
3164 net->ipv4.sysctl_tcp_sack = 1;
3165 net->ipv4.sysctl_tcp_window_scaling = 1;
3166 net->ipv4.sysctl_tcp_timestamps = 1;
3167 net->ipv4.sysctl_tcp_early_retrans = 3;
3168 net->ipv4.sysctl_tcp_recovery = TCP_RACK_LOSS_DETECTION;
3169 net->ipv4.sysctl_tcp_slow_start_after_idle = 1; /* By default, RFC2861 behavior. */
3170 net->ipv4.sysctl_tcp_retrans_collapse = 1;
3171 net->ipv4.sysctl_tcp_max_reordering = 300;
3172 net->ipv4.sysctl_tcp_dsack = 1;
3173 net->ipv4.sysctl_tcp_app_win = 31;
3174 net->ipv4.sysctl_tcp_adv_win_scale = 1;
3175 net->ipv4.sysctl_tcp_frto = 2;
3176 net->ipv4.sysctl_tcp_moderate_rcvbuf = 1;
3177 /* This limits the percentage of the congestion window which we
3178 * will allow a single TSO frame to consume. Building TSO frames
3179 * which are too large can cause TCP streams to be bursty.
3180 */
3181 net->ipv4.sysctl_tcp_tso_win_divisor = 3;
3182 /* Default TSQ limit of 16 TSO segments */
3183 net->ipv4.sysctl_tcp_limit_output_bytes = 16 * 65536;
3184
3185 /* rfc5961 challenge ack rate limiting, per net-ns, disabled by default. */
3186 net->ipv4.sysctl_tcp_challenge_ack_limit = INT_MAX;
3187
3188 net->ipv4.sysctl_tcp_min_tso_segs = 2;
3189 net->ipv4.sysctl_tcp_tso_rtt_log = 9; /* 2^9 = 512 usec */
3190 net->ipv4.sysctl_tcp_min_rtt_wlen = 300;
3191 net->ipv4.sysctl_tcp_autocorking = 1;
3192 net->ipv4.sysctl_tcp_invalid_ratelimit = HZ/2;
3193 net->ipv4.sysctl_tcp_pacing_ss_ratio = 200;
3194 net->ipv4.sysctl_tcp_pacing_ca_ratio = 120;
3195 if (net != &init_net) {
3196 memcpy(net->ipv4.sysctl_tcp_rmem,
3197 init_net.ipv4.sysctl_tcp_rmem,
3198 sizeof(init_net.ipv4.sysctl_tcp_rmem));
3199 memcpy(net->ipv4.sysctl_tcp_wmem,
3200 init_net.ipv4.sysctl_tcp_wmem,
3201 sizeof(init_net.ipv4.sysctl_tcp_wmem));
3202 }
3203 net->ipv4.sysctl_tcp_comp_sack_delay_ns = NSEC_PER_MSEC;
3204 net->ipv4.sysctl_tcp_comp_sack_slack_ns = 100 * NSEC_PER_USEC;
3205 net->ipv4.sysctl_tcp_comp_sack_nr = 44;
3206 net->ipv4.sysctl_tcp_fastopen = TFO_CLIENT_ENABLE;
3207 net->ipv4.sysctl_tcp_fastopen_blackhole_timeout = 0;
3208 atomic_set(&net->ipv4.tfo_active_disable_times, 0);
3209
3210 /* Reno is always built in */
3211 if (!net_eq(net, &init_net) &&
3212 bpf_try_module_get(init_net.ipv4.tcp_congestion_control,
3213 init_net.ipv4.tcp_congestion_control->owner))
3214 net->ipv4.tcp_congestion_control = init_net.ipv4.tcp_congestion_control;
3215 else
3216 net->ipv4.tcp_congestion_control = &tcp_reno;
3217
3218 return 0;
3219 }
3220
tcp_sk_exit_batch(struct list_head * net_exit_list)3221 static void __net_exit tcp_sk_exit_batch(struct list_head *net_exit_list)
3222 {
3223 struct net *net;
3224
3225 tcp_twsk_purge(net_exit_list, AF_INET);
3226
3227 list_for_each_entry(net, net_exit_list, exit_list) {
3228 inet_pernet_hashinfo_free(net->ipv4.tcp_death_row.hashinfo);
3229 WARN_ON_ONCE(!refcount_dec_and_test(&net->ipv4.tcp_death_row.tw_refcount));
3230 tcp_fastopen_ctx_destroy(net);
3231 }
3232 }
3233
3234 static struct pernet_operations __net_initdata tcp_sk_ops = {
3235 .init = tcp_sk_init,
3236 .exit = tcp_sk_exit,
3237 .exit_batch = tcp_sk_exit_batch,
3238 };
3239
3240 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
DEFINE_BPF_ITER_FUNC(tcp,struct bpf_iter_meta * meta,struct sock_common * sk_common,uid_t uid)3241 DEFINE_BPF_ITER_FUNC(tcp, struct bpf_iter_meta *meta,
3242 struct sock_common *sk_common, uid_t uid)
3243
3244 #define INIT_BATCH_SZ 16
3245
3246 static int bpf_iter_init_tcp(void *priv_data, struct bpf_iter_aux_info *aux)
3247 {
3248 struct bpf_tcp_iter_state *iter = priv_data;
3249 int err;
3250
3251 err = bpf_iter_init_seq_net(priv_data, aux);
3252 if (err)
3253 return err;
3254
3255 err = bpf_iter_tcp_realloc_batch(iter, INIT_BATCH_SZ);
3256 if (err) {
3257 bpf_iter_fini_seq_net(priv_data);
3258 return err;
3259 }
3260
3261 return 0;
3262 }
3263
bpf_iter_fini_tcp(void * priv_data)3264 static void bpf_iter_fini_tcp(void *priv_data)
3265 {
3266 struct bpf_tcp_iter_state *iter = priv_data;
3267
3268 bpf_iter_fini_seq_net(priv_data);
3269 kvfree(iter->batch);
3270 }
3271
3272 static const struct bpf_iter_seq_info tcp_seq_info = {
3273 .seq_ops = &bpf_iter_tcp_seq_ops,
3274 .init_seq_private = bpf_iter_init_tcp,
3275 .fini_seq_private = bpf_iter_fini_tcp,
3276 .seq_priv_size = sizeof(struct bpf_tcp_iter_state),
3277 };
3278
3279 static const struct bpf_func_proto *
bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,const struct bpf_prog * prog)3280 bpf_iter_tcp_get_func_proto(enum bpf_func_id func_id,
3281 const struct bpf_prog *prog)
3282 {
3283 switch (func_id) {
3284 case BPF_FUNC_setsockopt:
3285 return &bpf_sk_setsockopt_proto;
3286 case BPF_FUNC_getsockopt:
3287 return &bpf_sk_getsockopt_proto;
3288 default:
3289 return NULL;
3290 }
3291 }
3292
3293 static struct bpf_iter_reg tcp_reg_info = {
3294 .target = "tcp",
3295 .ctx_arg_info_size = 1,
3296 .ctx_arg_info = {
3297 { offsetof(struct bpf_iter__tcp, sk_common),
3298 PTR_TO_BTF_ID_OR_NULL },
3299 },
3300 .get_func_proto = bpf_iter_tcp_get_func_proto,
3301 .seq_info = &tcp_seq_info,
3302 };
3303
bpf_iter_register(void)3304 static void __init bpf_iter_register(void)
3305 {
3306 tcp_reg_info.ctx_arg_info[0].btf_id = btf_sock_ids[BTF_SOCK_TYPE_SOCK_COMMON];
3307 if (bpf_iter_reg_target(&tcp_reg_info))
3308 pr_warn("Warning: could not register bpf iterator tcp\n");
3309 }
3310
3311 #endif
3312
tcp_v4_init(void)3313 void __init tcp_v4_init(void)
3314 {
3315 int cpu, res;
3316
3317 for_each_possible_cpu(cpu) {
3318 struct sock *sk;
3319
3320 res = inet_ctl_sock_create(&sk, PF_INET, SOCK_RAW,
3321 IPPROTO_TCP, &init_net);
3322 if (res)
3323 panic("Failed to create the TCP control socket.\n");
3324 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
3325
3326 /* Please enforce IP_DF and IPID==0 for RST and
3327 * ACK sent in SYN-RECV and TIME-WAIT state.
3328 */
3329 inet_sk(sk)->pmtudisc = IP_PMTUDISC_DO;
3330
3331 per_cpu(ipv4_tcp_sk, cpu) = sk;
3332 }
3333 if (register_pernet_subsys(&tcp_sk_ops))
3334 panic("Failed to create the TCP control socket.\n");
3335
3336 #if defined(CONFIG_BPF_SYSCALL) && defined(CONFIG_PROC_FS)
3337 bpf_iter_register();
3338 #endif
3339 }
3340