1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * INET		An implementation of the TCP/IP protocol suite for the LINUX
4  *		operating system.  INET is implemented using the  BSD Socket
5  *		interface as the means of communication with the user level.
6  *
7  *		Generic socket support routines. Memory allocators, socket lock/release
8  *		handler for protocols to use and generic option handler.
9  *
10  * Authors:	Ross Biro
11  *		Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
12  *		Florian La Roche, <flla@stud.uni-sb.de>
13  *		Alan Cox, <A.Cox@swansea.ac.uk>
14  *
15  * Fixes:
16  *		Alan Cox	: 	Numerous verify_area() problems
17  *		Alan Cox	:	Connecting on a connecting socket
18  *					now returns an error for tcp.
19  *		Alan Cox	:	sock->protocol is set correctly.
20  *					and is not sometimes left as 0.
21  *		Alan Cox	:	connect handles icmp errors on a
22  *					connect properly. Unfortunately there
23  *					is a restart syscall nasty there. I
24  *					can't match BSD without hacking the C
25  *					library. Ideas urgently sought!
26  *		Alan Cox	:	Disallow bind() to addresses that are
27  *					not ours - especially broadcast ones!!
28  *		Alan Cox	:	Socket 1024 _IS_ ok for users. (fencepost)
29  *		Alan Cox	:	sock_wfree/sock_rfree don't destroy sockets,
30  *					instead they leave that for the DESTROY timer.
31  *		Alan Cox	:	Clean up error flag in accept
32  *		Alan Cox	:	TCP ack handling is buggy, the DESTROY timer
33  *					was buggy. Put a remove_sock() in the handler
34  *					for memory when we hit 0. Also altered the timer
35  *					code. The ACK stuff can wait and needs major
36  *					TCP layer surgery.
37  *		Alan Cox	:	Fixed TCP ack bug, removed remove sock
38  *					and fixed timer/inet_bh race.
39  *		Alan Cox	:	Added zapped flag for TCP
40  *		Alan Cox	:	Move kfree_skb into skbuff.c and tidied up surplus code
41  *		Alan Cox	:	for new sk_buff allocations wmalloc/rmalloc now call alloc_skb
42  *		Alan Cox	:	kfree_s calls now are kfree_skbmem so we can track skb resources
43  *		Alan Cox	:	Supports socket option broadcast now as does udp. Packet and raw need fixing.
44  *		Alan Cox	:	Added RCVBUF,SNDBUF size setting. It suddenly occurred to me how easy it was so...
45  *		Rick Sladkey	:	Relaxed UDP rules for matching packets.
46  *		C.E.Hawkins	:	IFF_PROMISC/SIOCGHWADDR support
47  *	Pauline Middelink	:	identd support
48  *		Alan Cox	:	Fixed connect() taking signals I think.
49  *		Alan Cox	:	SO_LINGER supported
50  *		Alan Cox	:	Error reporting fixes
51  *		Anonymous	:	inet_create tidied up (sk->reuse setting)
52  *		Alan Cox	:	inet sockets don't set sk->type!
53  *		Alan Cox	:	Split socket option code
54  *		Alan Cox	:	Callbacks
55  *		Alan Cox	:	Nagle flag for Charles & Johannes stuff
56  *		Alex		:	Removed restriction on inet fioctl
57  *		Alan Cox	:	Splitting INET from NET core
58  *		Alan Cox	:	Fixed bogus SO_TYPE handling in getsockopt()
59  *		Adam Caldwell	:	Missing return in SO_DONTROUTE/SO_DEBUG code
60  *		Alan Cox	:	Split IP from generic code
61  *		Alan Cox	:	New kfree_skbmem()
62  *		Alan Cox	:	Make SO_DEBUG superuser only.
63  *		Alan Cox	:	Allow anyone to clear SO_DEBUG
64  *					(compatibility fix)
65  *		Alan Cox	:	Added optimistic memory grabbing for AF_UNIX throughput.
66  *		Alan Cox	:	Allocator for a socket is settable.
67  *		Alan Cox	:	SO_ERROR includes soft errors.
68  *		Alan Cox	:	Allow NULL arguments on some SO_ opts
69  *		Alan Cox	: 	Generic socket allocation to make hooks
70  *					easier (suggested by Craig Metz).
71  *		Michael Pall	:	SO_ERROR returns positive errno again
72  *              Steve Whitehouse:       Added default destructor to free
73  *                                      protocol private data.
74  *              Steve Whitehouse:       Added various other default routines
75  *                                      common to several socket families.
76  *              Chris Evans     :       Call suser() check last on F_SETOWN
77  *		Jay Schulist	:	Added SO_ATTACH_FILTER and SO_DETACH_FILTER.
78  *		Andi Kleen	:	Add sock_kmalloc()/sock_kfree_s()
79  *		Andi Kleen	:	Fix write_space callback
80  *		Chris Evans	:	Security fixes - signedness again
81  *		Arnaldo C. Melo :       cleanups, use skb_queue_purge
82  *
83  * To Fix:
84  */
85 
86 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
87 
88 #include <asm/unaligned.h>
89 #include <linux/capability.h>
90 #include <linux/errno.h>
91 #include <linux/errqueue.h>
92 #include <linux/types.h>
93 #include <linux/socket.h>
94 #include <linux/in.h>
95 #include <linux/kernel.h>
96 #include <linux/module.h>
97 #include <linux/proc_fs.h>
98 #include <linux/seq_file.h>
99 #include <linux/sched.h>
100 #include <linux/sched/mm.h>
101 #include <linux/timer.h>
102 #include <linux/string.h>
103 #include <linux/sockios.h>
104 #include <linux/net.h>
105 #include <linux/mm.h>
106 #include <linux/slab.h>
107 #include <linux/interrupt.h>
108 #include <linux/poll.h>
109 #include <linux/tcp.h>
110 #include <linux/init.h>
111 #include <linux/highmem.h>
112 #include <linux/user_namespace.h>
113 #include <linux/static_key.h>
114 #include <linux/memcontrol.h>
115 #include <linux/prefetch.h>
116 #include <linux/compat.h>
117 
118 #include <linux/uaccess.h>
119 
120 #include <linux/netdevice.h>
121 #include <net/protocol.h>
122 #include <linux/skbuff.h>
123 #include <net/net_namespace.h>
124 #include <net/request_sock.h>
125 #include <net/sock.h>
126 #include <linux/net_tstamp.h>
127 #include <net/xfrm.h>
128 #include <linux/ipsec.h>
129 #include <net/cls_cgroup.h>
130 #include <net/netprio_cgroup.h>
131 #include <linux/sock_diag.h>
132 
133 #include <linux/filter.h>
134 #include <net/sock_reuseport.h>
135 #include <net/bpf_sk_storage.h>
136 
137 #include <trace/events/sock.h>
138 
139 #include <net/tcp.h>
140 #include <net/busy_poll.h>
141 
142 #include <linux/ethtool.h>
143 
144 #include "dev.h"
145 
146 static DEFINE_MUTEX(proto_list_mutex);
147 static LIST_HEAD(proto_list);
148 
149 static void sock_def_write_space_wfree(struct sock *sk);
150 static void sock_def_write_space(struct sock *sk);
151 
152 /**
153  * sk_ns_capable - General socket capability test
154  * @sk: Socket to use a capability on or through
155  * @user_ns: The user namespace of the capability to use
156  * @cap: The capability to use
157  *
158  * Test to see if the opener of the socket had when the socket was
159  * created and the current process has the capability @cap in the user
160  * namespace @user_ns.
161  */
sk_ns_capable(const struct sock * sk,struct user_namespace * user_ns,int cap)162 bool sk_ns_capable(const struct sock *sk,
163 		   struct user_namespace *user_ns, int cap)
164 {
165 	return file_ns_capable(sk->sk_socket->file, user_ns, cap) &&
166 		ns_capable(user_ns, cap);
167 }
168 EXPORT_SYMBOL(sk_ns_capable);
169 
170 /**
171  * sk_capable - Socket global capability test
172  * @sk: Socket to use a capability on or through
173  * @cap: The global capability to use
174  *
175  * Test to see if the opener of the socket had when the socket was
176  * created and the current process has the capability @cap in all user
177  * namespaces.
178  */
sk_capable(const struct sock * sk,int cap)179 bool sk_capable(const struct sock *sk, int cap)
180 {
181 	return sk_ns_capable(sk, &init_user_ns, cap);
182 }
183 EXPORT_SYMBOL(sk_capable);
184 
185 /**
186  * sk_net_capable - Network namespace socket capability test
187  * @sk: Socket to use a capability on or through
188  * @cap: The capability to use
189  *
190  * Test to see if the opener of the socket had when the socket was created
191  * and the current process has the capability @cap over the network namespace
192  * the socket is a member of.
193  */
sk_net_capable(const struct sock * sk,int cap)194 bool sk_net_capable(const struct sock *sk, int cap)
195 {
196 	return sk_ns_capable(sk, sock_net(sk)->user_ns, cap);
197 }
198 EXPORT_SYMBOL(sk_net_capable);
199 
200 /*
201  * Each address family might have different locking rules, so we have
202  * one slock key per address family and separate keys for internal and
203  * userspace sockets.
204  */
205 static struct lock_class_key af_family_keys[AF_MAX];
206 static struct lock_class_key af_family_kern_keys[AF_MAX];
207 static struct lock_class_key af_family_slock_keys[AF_MAX];
208 static struct lock_class_key af_family_kern_slock_keys[AF_MAX];
209 
210 /*
211  * Make lock validator output more readable. (we pre-construct these
212  * strings build-time, so that runtime initialization of socket
213  * locks is fast):
214  */
215 
216 #define _sock_locks(x)						  \
217   x "AF_UNSPEC",	x "AF_UNIX"     ,	x "AF_INET"     , \
218   x "AF_AX25"  ,	x "AF_IPX"      ,	x "AF_APPLETALK", \
219   x "AF_NETROM",	x "AF_BRIDGE"   ,	x "AF_ATMPVC"   , \
220   x "AF_X25"   ,	x "AF_INET6"    ,	x "AF_ROSE"     , \
221   x "AF_DECnet",	x "AF_NETBEUI"  ,	x "AF_SECURITY" , \
222   x "AF_KEY"   ,	x "AF_NETLINK"  ,	x "AF_PACKET"   , \
223   x "AF_ASH"   ,	x "AF_ECONET"   ,	x "AF_ATMSVC"   , \
224   x "AF_RDS"   ,	x "AF_SNA"      ,	x "AF_IRDA"     , \
225   x "AF_PPPOX" ,	x "AF_WANPIPE"  ,	x "AF_LLC"      , \
226   x "27"       ,	x "28"          ,	x "AF_CAN"      , \
227   x "AF_TIPC"  ,	x "AF_BLUETOOTH",	x "IUCV"        , \
228   x "AF_RXRPC" ,	x "AF_ISDN"     ,	x "AF_PHONET"   , \
229   x "AF_IEEE802154",	x "AF_CAIF"	,	x "AF_ALG"      , \
230   x "AF_NFC"   ,	x "AF_VSOCK"    ,	x "AF_KCM"      , \
231   x "AF_QIPCRTR",	x "AF_SMC"	,	x "AF_XDP"	, \
232   x "AF_MCTP"  , \
233   x "AF_MAX"
234 
235 static const char *const af_family_key_strings[AF_MAX+1] = {
236 	_sock_locks("sk_lock-")
237 };
238 static const char *const af_family_slock_key_strings[AF_MAX+1] = {
239 	_sock_locks("slock-")
240 };
241 static const char *const af_family_clock_key_strings[AF_MAX+1] = {
242 	_sock_locks("clock-")
243 };
244 
245 static const char *const af_family_kern_key_strings[AF_MAX+1] = {
246 	_sock_locks("k-sk_lock-")
247 };
248 static const char *const af_family_kern_slock_key_strings[AF_MAX+1] = {
249 	_sock_locks("k-slock-")
250 };
251 static const char *const af_family_kern_clock_key_strings[AF_MAX+1] = {
252 	_sock_locks("k-clock-")
253 };
254 static const char *const af_family_rlock_key_strings[AF_MAX+1] = {
255 	_sock_locks("rlock-")
256 };
257 static const char *const af_family_wlock_key_strings[AF_MAX+1] = {
258 	_sock_locks("wlock-")
259 };
260 static const char *const af_family_elock_key_strings[AF_MAX+1] = {
261 	_sock_locks("elock-")
262 };
263 
264 /*
265  * sk_callback_lock and sk queues locking rules are per-address-family,
266  * so split the lock classes by using a per-AF key:
267  */
268 static struct lock_class_key af_callback_keys[AF_MAX];
269 static struct lock_class_key af_rlock_keys[AF_MAX];
270 static struct lock_class_key af_wlock_keys[AF_MAX];
271 static struct lock_class_key af_elock_keys[AF_MAX];
272 static struct lock_class_key af_kern_callback_keys[AF_MAX];
273 
274 /* Run time adjustable parameters. */
275 __u32 sysctl_wmem_max __read_mostly = SK_WMEM_MAX;
276 EXPORT_SYMBOL(sysctl_wmem_max);
277 __u32 sysctl_rmem_max __read_mostly = SK_RMEM_MAX;
278 EXPORT_SYMBOL(sysctl_rmem_max);
279 __u32 sysctl_wmem_default __read_mostly = SK_WMEM_MAX;
280 __u32 sysctl_rmem_default __read_mostly = SK_RMEM_MAX;
281 
282 /* Maximal space eaten by iovec or ancillary data plus some space */
283 int sysctl_optmem_max __read_mostly = sizeof(unsigned long)*(2*UIO_MAXIOV+512);
284 EXPORT_SYMBOL(sysctl_optmem_max);
285 
286 int sysctl_tstamp_allow_data __read_mostly = 1;
287 
288 DEFINE_STATIC_KEY_FALSE(memalloc_socks_key);
289 EXPORT_SYMBOL_GPL(memalloc_socks_key);
290 
291 /**
292  * sk_set_memalloc - sets %SOCK_MEMALLOC
293  * @sk: socket to set it on
294  *
295  * Set %SOCK_MEMALLOC on a socket for access to emergency reserves.
296  * It's the responsibility of the admin to adjust min_free_kbytes
297  * to meet the requirements
298  */
sk_set_memalloc(struct sock * sk)299 void sk_set_memalloc(struct sock *sk)
300 {
301 	sock_set_flag(sk, SOCK_MEMALLOC);
302 	sk->sk_allocation |= __GFP_MEMALLOC;
303 	static_branch_inc(&memalloc_socks_key);
304 }
305 EXPORT_SYMBOL_GPL(sk_set_memalloc);
306 
sk_clear_memalloc(struct sock * sk)307 void sk_clear_memalloc(struct sock *sk)
308 {
309 	sock_reset_flag(sk, SOCK_MEMALLOC);
310 	sk->sk_allocation &= ~__GFP_MEMALLOC;
311 	static_branch_dec(&memalloc_socks_key);
312 
313 	/*
314 	 * SOCK_MEMALLOC is allowed to ignore rmem limits to ensure forward
315 	 * progress of swapping. SOCK_MEMALLOC may be cleared while
316 	 * it has rmem allocations due to the last swapfile being deactivated
317 	 * but there is a risk that the socket is unusable due to exceeding
318 	 * the rmem limits. Reclaim the reserves and obey rmem limits again.
319 	 */
320 	sk_mem_reclaim(sk);
321 }
322 EXPORT_SYMBOL_GPL(sk_clear_memalloc);
323 
__sk_backlog_rcv(struct sock * sk,struct sk_buff * skb)324 int __sk_backlog_rcv(struct sock *sk, struct sk_buff *skb)
325 {
326 	int ret;
327 	unsigned int noreclaim_flag;
328 
329 	/* these should have been dropped before queueing */
330 	BUG_ON(!sock_flag(sk, SOCK_MEMALLOC));
331 
332 	noreclaim_flag = memalloc_noreclaim_save();
333 	ret = INDIRECT_CALL_INET(sk->sk_backlog_rcv,
334 				 tcp_v6_do_rcv,
335 				 tcp_v4_do_rcv,
336 				 sk, skb);
337 	memalloc_noreclaim_restore(noreclaim_flag);
338 
339 	return ret;
340 }
341 EXPORT_SYMBOL(__sk_backlog_rcv);
342 
sk_error_report(struct sock * sk)343 void sk_error_report(struct sock *sk)
344 {
345 	sk->sk_error_report(sk);
346 
347 	switch (sk->sk_family) {
348 	case AF_INET:
349 		fallthrough;
350 	case AF_INET6:
351 		trace_inet_sk_error_report(sk);
352 		break;
353 	default:
354 		break;
355 	}
356 }
357 EXPORT_SYMBOL(sk_error_report);
358 
sock_get_timeout(long timeo,void * optval,bool old_timeval)359 int sock_get_timeout(long timeo, void *optval, bool old_timeval)
360 {
361 	struct __kernel_sock_timeval tv;
362 
363 	if (timeo == MAX_SCHEDULE_TIMEOUT) {
364 		tv.tv_sec = 0;
365 		tv.tv_usec = 0;
366 	} else {
367 		tv.tv_sec = timeo / HZ;
368 		tv.tv_usec = ((timeo % HZ) * USEC_PER_SEC) / HZ;
369 	}
370 
371 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
372 		struct old_timeval32 tv32 = { tv.tv_sec, tv.tv_usec };
373 		*(struct old_timeval32 *)optval = tv32;
374 		return sizeof(tv32);
375 	}
376 
377 	if (old_timeval) {
378 		struct __kernel_old_timeval old_tv;
379 		old_tv.tv_sec = tv.tv_sec;
380 		old_tv.tv_usec = tv.tv_usec;
381 		*(struct __kernel_old_timeval *)optval = old_tv;
382 		return sizeof(old_tv);
383 	}
384 
385 	*(struct __kernel_sock_timeval *)optval = tv;
386 	return sizeof(tv);
387 }
388 EXPORT_SYMBOL(sock_get_timeout);
389 
sock_copy_user_timeval(struct __kernel_sock_timeval * tv,sockptr_t optval,int optlen,bool old_timeval)390 int sock_copy_user_timeval(struct __kernel_sock_timeval *tv,
391 			   sockptr_t optval, int optlen, bool old_timeval)
392 {
393 	if (old_timeval && in_compat_syscall() && !COMPAT_USE_64BIT_TIME) {
394 		struct old_timeval32 tv32;
395 
396 		if (optlen < sizeof(tv32))
397 			return -EINVAL;
398 
399 		if (copy_from_sockptr(&tv32, optval, sizeof(tv32)))
400 			return -EFAULT;
401 		tv->tv_sec = tv32.tv_sec;
402 		tv->tv_usec = tv32.tv_usec;
403 	} else if (old_timeval) {
404 		struct __kernel_old_timeval old_tv;
405 
406 		if (optlen < sizeof(old_tv))
407 			return -EINVAL;
408 		if (copy_from_sockptr(&old_tv, optval, sizeof(old_tv)))
409 			return -EFAULT;
410 		tv->tv_sec = old_tv.tv_sec;
411 		tv->tv_usec = old_tv.tv_usec;
412 	} else {
413 		if (optlen < sizeof(*tv))
414 			return -EINVAL;
415 		if (copy_from_sockptr(tv, optval, sizeof(*tv)))
416 			return -EFAULT;
417 	}
418 
419 	return 0;
420 }
421 EXPORT_SYMBOL(sock_copy_user_timeval);
422 
sock_set_timeout(long * timeo_p,sockptr_t optval,int optlen,bool old_timeval)423 static int sock_set_timeout(long *timeo_p, sockptr_t optval, int optlen,
424 			    bool old_timeval)
425 {
426 	struct __kernel_sock_timeval tv;
427 	int err = sock_copy_user_timeval(&tv, optval, optlen, old_timeval);
428 
429 	if (err)
430 		return err;
431 
432 	if (tv.tv_usec < 0 || tv.tv_usec >= USEC_PER_SEC)
433 		return -EDOM;
434 
435 	if (tv.tv_sec < 0) {
436 		static int warned __read_mostly;
437 
438 		*timeo_p = 0;
439 		if (warned < 10 && net_ratelimit()) {
440 			warned++;
441 			pr_info("%s: `%s' (pid %d) tries to set negative timeout\n",
442 				__func__, current->comm, task_pid_nr(current));
443 		}
444 		return 0;
445 	}
446 	*timeo_p = MAX_SCHEDULE_TIMEOUT;
447 	if (tv.tv_sec == 0 && tv.tv_usec == 0)
448 		return 0;
449 	if (tv.tv_sec < (MAX_SCHEDULE_TIMEOUT / HZ - 1))
450 		*timeo_p = tv.tv_sec * HZ + DIV_ROUND_UP((unsigned long)tv.tv_usec, USEC_PER_SEC / HZ);
451 	return 0;
452 }
453 
sock_needs_netstamp(const struct sock * sk)454 static bool sock_needs_netstamp(const struct sock *sk)
455 {
456 	switch (sk->sk_family) {
457 	case AF_UNSPEC:
458 	case AF_UNIX:
459 		return false;
460 	default:
461 		return true;
462 	}
463 }
464 
sock_disable_timestamp(struct sock * sk,unsigned long flags)465 static void sock_disable_timestamp(struct sock *sk, unsigned long flags)
466 {
467 	if (sk->sk_flags & flags) {
468 		sk->sk_flags &= ~flags;
469 		if (sock_needs_netstamp(sk) &&
470 		    !(sk->sk_flags & SK_FLAGS_TIMESTAMP))
471 			net_disable_timestamp();
472 	}
473 }
474 
475 
__sock_queue_rcv_skb(struct sock * sk,struct sk_buff * skb)476 int __sock_queue_rcv_skb(struct sock *sk, struct sk_buff *skb)
477 {
478 	unsigned long flags;
479 	struct sk_buff_head *list = &sk->sk_receive_queue;
480 
481 	if (atomic_read(&sk->sk_rmem_alloc) >= sk->sk_rcvbuf) {
482 		atomic_inc(&sk->sk_drops);
483 		trace_sock_rcvqueue_full(sk, skb);
484 		return -ENOMEM;
485 	}
486 
487 	if (!sk_rmem_schedule(sk, skb, skb->truesize)) {
488 		atomic_inc(&sk->sk_drops);
489 		return -ENOBUFS;
490 	}
491 
492 	skb->dev = NULL;
493 	skb_set_owner_r(skb, sk);
494 
495 	/* we escape from rcu protected region, make sure we dont leak
496 	 * a norefcounted dst
497 	 */
498 	skb_dst_force(skb);
499 
500 	spin_lock_irqsave(&list->lock, flags);
501 	sock_skb_set_dropcount(sk, skb);
502 	__skb_queue_tail(list, skb);
503 	spin_unlock_irqrestore(&list->lock, flags);
504 
505 	if (!sock_flag(sk, SOCK_DEAD))
506 		sk->sk_data_ready(sk);
507 	return 0;
508 }
509 EXPORT_SYMBOL(__sock_queue_rcv_skb);
510 
sock_queue_rcv_skb_reason(struct sock * sk,struct sk_buff * skb,enum skb_drop_reason * reason)511 int sock_queue_rcv_skb_reason(struct sock *sk, struct sk_buff *skb,
512 			      enum skb_drop_reason *reason)
513 {
514 	enum skb_drop_reason drop_reason;
515 	int err;
516 
517 	err = sk_filter(sk, skb);
518 	if (err) {
519 		drop_reason = SKB_DROP_REASON_SOCKET_FILTER;
520 		goto out;
521 	}
522 	err = __sock_queue_rcv_skb(sk, skb);
523 	switch (err) {
524 	case -ENOMEM:
525 		drop_reason = SKB_DROP_REASON_SOCKET_RCVBUFF;
526 		break;
527 	case -ENOBUFS:
528 		drop_reason = SKB_DROP_REASON_PROTO_MEM;
529 		break;
530 	default:
531 		drop_reason = SKB_NOT_DROPPED_YET;
532 		break;
533 	}
534 out:
535 	if (reason)
536 		*reason = drop_reason;
537 	return err;
538 }
539 EXPORT_SYMBOL(sock_queue_rcv_skb_reason);
540 
__sk_receive_skb(struct sock * sk,struct sk_buff * skb,const int nested,unsigned int trim_cap,bool refcounted)541 int __sk_receive_skb(struct sock *sk, struct sk_buff *skb,
542 		     const int nested, unsigned int trim_cap, bool refcounted)
543 {
544 	int rc = NET_RX_SUCCESS;
545 
546 	if (sk_filter_trim_cap(sk, skb, trim_cap))
547 		goto discard_and_relse;
548 
549 	skb->dev = NULL;
550 
551 	if (sk_rcvqueues_full(sk, sk->sk_rcvbuf)) {
552 		atomic_inc(&sk->sk_drops);
553 		goto discard_and_relse;
554 	}
555 	if (nested)
556 		bh_lock_sock_nested(sk);
557 	else
558 		bh_lock_sock(sk);
559 	if (!sock_owned_by_user(sk)) {
560 		/*
561 		 * trylock + unlock semantics:
562 		 */
563 		mutex_acquire(&sk->sk_lock.dep_map, 0, 1, _RET_IP_);
564 
565 		rc = sk_backlog_rcv(sk, skb);
566 
567 		mutex_release(&sk->sk_lock.dep_map, _RET_IP_);
568 	} else if (sk_add_backlog(sk, skb, READ_ONCE(sk->sk_rcvbuf))) {
569 		bh_unlock_sock(sk);
570 		atomic_inc(&sk->sk_drops);
571 		goto discard_and_relse;
572 	}
573 
574 	bh_unlock_sock(sk);
575 out:
576 	if (refcounted)
577 		sock_put(sk);
578 	return rc;
579 discard_and_relse:
580 	kfree_skb(skb);
581 	goto out;
582 }
583 EXPORT_SYMBOL(__sk_receive_skb);
584 
585 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ip6_dst_check(struct dst_entry *,
586 							  u32));
587 INDIRECT_CALLABLE_DECLARE(struct dst_entry *ipv4_dst_check(struct dst_entry *,
588 							   u32));
__sk_dst_check(struct sock * sk,u32 cookie)589 struct dst_entry *__sk_dst_check(struct sock *sk, u32 cookie)
590 {
591 	struct dst_entry *dst = __sk_dst_get(sk);
592 
593 	if (dst && dst->obsolete &&
594 	    INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
595 			       dst, cookie) == NULL) {
596 		sk_tx_queue_clear(sk);
597 		sk->sk_dst_pending_confirm = 0;
598 		RCU_INIT_POINTER(sk->sk_dst_cache, NULL);
599 		dst_release(dst);
600 		return NULL;
601 	}
602 
603 	return dst;
604 }
605 EXPORT_SYMBOL(__sk_dst_check);
606 
sk_dst_check(struct sock * sk,u32 cookie)607 struct dst_entry *sk_dst_check(struct sock *sk, u32 cookie)
608 {
609 	struct dst_entry *dst = sk_dst_get(sk);
610 
611 	if (dst && dst->obsolete &&
612 	    INDIRECT_CALL_INET(dst->ops->check, ip6_dst_check, ipv4_dst_check,
613 			       dst, cookie) == NULL) {
614 		sk_dst_reset(sk);
615 		dst_release(dst);
616 		return NULL;
617 	}
618 
619 	return dst;
620 }
621 EXPORT_SYMBOL(sk_dst_check);
622 
sock_bindtoindex_locked(struct sock * sk,int ifindex)623 static int sock_bindtoindex_locked(struct sock *sk, int ifindex)
624 {
625 	int ret = -ENOPROTOOPT;
626 #ifdef CONFIG_NETDEVICES
627 	struct net *net = sock_net(sk);
628 
629 	/* Sorry... */
630 	ret = -EPERM;
631 	if (sk->sk_bound_dev_if && !ns_capable(net->user_ns, CAP_NET_RAW))
632 		goto out;
633 
634 	ret = -EINVAL;
635 	if (ifindex < 0)
636 		goto out;
637 
638 	/* Paired with all READ_ONCE() done locklessly. */
639 	WRITE_ONCE(sk->sk_bound_dev_if, ifindex);
640 
641 	if (sk->sk_prot->rehash)
642 		sk->sk_prot->rehash(sk);
643 	sk_dst_reset(sk);
644 
645 	ret = 0;
646 
647 out:
648 #endif
649 
650 	return ret;
651 }
652 
sock_bindtoindex(struct sock * sk,int ifindex,bool lock_sk)653 int sock_bindtoindex(struct sock *sk, int ifindex, bool lock_sk)
654 {
655 	int ret;
656 
657 	if (lock_sk)
658 		lock_sock(sk);
659 	ret = sock_bindtoindex_locked(sk, ifindex);
660 	if (lock_sk)
661 		release_sock(sk);
662 
663 	return ret;
664 }
665 EXPORT_SYMBOL(sock_bindtoindex);
666 
sock_setbindtodevice(struct sock * sk,sockptr_t optval,int optlen)667 static int sock_setbindtodevice(struct sock *sk, sockptr_t optval, int optlen)
668 {
669 	int ret = -ENOPROTOOPT;
670 #ifdef CONFIG_NETDEVICES
671 	struct net *net = sock_net(sk);
672 	char devname[IFNAMSIZ];
673 	int index;
674 
675 	ret = -EINVAL;
676 	if (optlen < 0)
677 		goto out;
678 
679 	/* Bind this socket to a particular device like "eth0",
680 	 * as specified in the passed interface name. If the
681 	 * name is "" or the option length is zero the socket
682 	 * is not bound.
683 	 */
684 	if (optlen > IFNAMSIZ - 1)
685 		optlen = IFNAMSIZ - 1;
686 	memset(devname, 0, sizeof(devname));
687 
688 	ret = -EFAULT;
689 	if (copy_from_sockptr(devname, optval, optlen))
690 		goto out;
691 
692 	index = 0;
693 	if (devname[0] != '\0') {
694 		struct net_device *dev;
695 
696 		rcu_read_lock();
697 		dev = dev_get_by_name_rcu(net, devname);
698 		if (dev)
699 			index = dev->ifindex;
700 		rcu_read_unlock();
701 		ret = -ENODEV;
702 		if (!dev)
703 			goto out;
704 	}
705 
706 	sockopt_lock_sock(sk);
707 	ret = sock_bindtoindex_locked(sk, index);
708 	sockopt_release_sock(sk);
709 out:
710 #endif
711 
712 	return ret;
713 }
714 
sock_getbindtodevice(struct sock * sk,sockptr_t optval,sockptr_t optlen,int len)715 static int sock_getbindtodevice(struct sock *sk, sockptr_t optval,
716 				sockptr_t optlen, int len)
717 {
718 	int ret = -ENOPROTOOPT;
719 #ifdef CONFIG_NETDEVICES
720 	int bound_dev_if = READ_ONCE(sk->sk_bound_dev_if);
721 	struct net *net = sock_net(sk);
722 	char devname[IFNAMSIZ];
723 
724 	if (bound_dev_if == 0) {
725 		len = 0;
726 		goto zero;
727 	}
728 
729 	ret = -EINVAL;
730 	if (len < IFNAMSIZ)
731 		goto out;
732 
733 	ret = netdev_get_name(net, devname, bound_dev_if);
734 	if (ret)
735 		goto out;
736 
737 	len = strlen(devname) + 1;
738 
739 	ret = -EFAULT;
740 	if (copy_to_sockptr(optval, devname, len))
741 		goto out;
742 
743 zero:
744 	ret = -EFAULT;
745 	if (copy_to_sockptr(optlen, &len, sizeof(int)))
746 		goto out;
747 
748 	ret = 0;
749 
750 out:
751 #endif
752 
753 	return ret;
754 }
755 
sk_mc_loop(struct sock * sk)756 bool sk_mc_loop(struct sock *sk)
757 {
758 	if (dev_recursion_level())
759 		return false;
760 	if (!sk)
761 		return true;
762 	switch (sk->sk_family) {
763 	case AF_INET:
764 		return inet_sk(sk)->mc_loop;
765 #if IS_ENABLED(CONFIG_IPV6)
766 	case AF_INET6:
767 		return inet6_sk(sk)->mc_loop;
768 #endif
769 	}
770 	WARN_ON_ONCE(1);
771 	return true;
772 }
773 EXPORT_SYMBOL(sk_mc_loop);
774 
sock_set_reuseaddr(struct sock * sk)775 void sock_set_reuseaddr(struct sock *sk)
776 {
777 	lock_sock(sk);
778 	sk->sk_reuse = SK_CAN_REUSE;
779 	release_sock(sk);
780 }
781 EXPORT_SYMBOL(sock_set_reuseaddr);
782 
sock_set_reuseport(struct sock * sk)783 void sock_set_reuseport(struct sock *sk)
784 {
785 	lock_sock(sk);
786 	sk->sk_reuseport = true;
787 	release_sock(sk);
788 }
789 EXPORT_SYMBOL(sock_set_reuseport);
790 
sock_no_linger(struct sock * sk)791 void sock_no_linger(struct sock *sk)
792 {
793 	lock_sock(sk);
794 	sk->sk_lingertime = 0;
795 	sock_set_flag(sk, SOCK_LINGER);
796 	release_sock(sk);
797 }
798 EXPORT_SYMBOL(sock_no_linger);
799 
sock_set_priority(struct sock * sk,u32 priority)800 void sock_set_priority(struct sock *sk, u32 priority)
801 {
802 	lock_sock(sk);
803 	sk->sk_priority = priority;
804 	release_sock(sk);
805 }
806 EXPORT_SYMBOL(sock_set_priority);
807 
sock_set_sndtimeo(struct sock * sk,s64 secs)808 void sock_set_sndtimeo(struct sock *sk, s64 secs)
809 {
810 	lock_sock(sk);
811 	if (secs && secs < MAX_SCHEDULE_TIMEOUT / HZ - 1)
812 		sk->sk_sndtimeo = secs * HZ;
813 	else
814 		sk->sk_sndtimeo = MAX_SCHEDULE_TIMEOUT;
815 	release_sock(sk);
816 }
817 EXPORT_SYMBOL(sock_set_sndtimeo);
818 
__sock_set_timestamps(struct sock * sk,bool val,bool new,bool ns)819 static void __sock_set_timestamps(struct sock *sk, bool val, bool new, bool ns)
820 {
821 	if (val)  {
822 		sock_valbool_flag(sk, SOCK_TSTAMP_NEW, new);
823 		sock_valbool_flag(sk, SOCK_RCVTSTAMPNS, ns);
824 		sock_set_flag(sk, SOCK_RCVTSTAMP);
825 		sock_enable_timestamp(sk, SOCK_TIMESTAMP);
826 	} else {
827 		sock_reset_flag(sk, SOCK_RCVTSTAMP);
828 		sock_reset_flag(sk, SOCK_RCVTSTAMPNS);
829 	}
830 }
831 
sock_enable_timestamps(struct sock * sk)832 void sock_enable_timestamps(struct sock *sk)
833 {
834 	lock_sock(sk);
835 	__sock_set_timestamps(sk, true, false, true);
836 	release_sock(sk);
837 }
838 EXPORT_SYMBOL(sock_enable_timestamps);
839 
sock_set_timestamp(struct sock * sk,int optname,bool valbool)840 void sock_set_timestamp(struct sock *sk, int optname, bool valbool)
841 {
842 	switch (optname) {
843 	case SO_TIMESTAMP_OLD:
844 		__sock_set_timestamps(sk, valbool, false, false);
845 		break;
846 	case SO_TIMESTAMP_NEW:
847 		__sock_set_timestamps(sk, valbool, true, false);
848 		break;
849 	case SO_TIMESTAMPNS_OLD:
850 		__sock_set_timestamps(sk, valbool, false, true);
851 		break;
852 	case SO_TIMESTAMPNS_NEW:
853 		__sock_set_timestamps(sk, valbool, true, true);
854 		break;
855 	}
856 }
857 
sock_timestamping_bind_phc(struct sock * sk,int phc_index)858 static int sock_timestamping_bind_phc(struct sock *sk, int phc_index)
859 {
860 	struct net *net = sock_net(sk);
861 	struct net_device *dev = NULL;
862 	bool match = false;
863 	int *vclock_index;
864 	int i, num;
865 
866 	if (sk->sk_bound_dev_if)
867 		dev = dev_get_by_index(net, sk->sk_bound_dev_if);
868 
869 	if (!dev) {
870 		pr_err("%s: sock not bind to device\n", __func__);
871 		return -EOPNOTSUPP;
872 	}
873 
874 	num = ethtool_get_phc_vclocks(dev, &vclock_index);
875 	dev_put(dev);
876 
877 	for (i = 0; i < num; i++) {
878 		if (*(vclock_index + i) == phc_index) {
879 			match = true;
880 			break;
881 		}
882 	}
883 
884 	if (num > 0)
885 		kfree(vclock_index);
886 
887 	if (!match)
888 		return -EINVAL;
889 
890 	sk->sk_bind_phc = phc_index;
891 
892 	return 0;
893 }
894 
sock_set_timestamping(struct sock * sk,int optname,struct so_timestamping timestamping)895 int sock_set_timestamping(struct sock *sk, int optname,
896 			  struct so_timestamping timestamping)
897 {
898 	int val = timestamping.flags;
899 	int ret;
900 
901 	if (val & ~SOF_TIMESTAMPING_MASK)
902 		return -EINVAL;
903 
904 	if (val & SOF_TIMESTAMPING_OPT_ID &&
905 	    !(sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)) {
906 		if (sk_is_tcp(sk)) {
907 			if ((1 << sk->sk_state) &
908 			    (TCPF_CLOSE | TCPF_LISTEN))
909 				return -EINVAL;
910 			atomic_set(&sk->sk_tskey, tcp_sk(sk)->snd_una);
911 		} else {
912 			atomic_set(&sk->sk_tskey, 0);
913 		}
914 	}
915 
916 	if (val & SOF_TIMESTAMPING_OPT_STATS &&
917 	    !(val & SOF_TIMESTAMPING_OPT_TSONLY))
918 		return -EINVAL;
919 
920 	if (val & SOF_TIMESTAMPING_BIND_PHC) {
921 		ret = sock_timestamping_bind_phc(sk, timestamping.bind_phc);
922 		if (ret)
923 			return ret;
924 	}
925 
926 	sk->sk_tsflags = val;
927 	sock_valbool_flag(sk, SOCK_TSTAMP_NEW, optname == SO_TIMESTAMPING_NEW);
928 
929 	if (val & SOF_TIMESTAMPING_RX_SOFTWARE)
930 		sock_enable_timestamp(sk,
931 				      SOCK_TIMESTAMPING_RX_SOFTWARE);
932 	else
933 		sock_disable_timestamp(sk,
934 				       (1UL << SOCK_TIMESTAMPING_RX_SOFTWARE));
935 	return 0;
936 }
937 
sock_set_keepalive(struct sock * sk)938 void sock_set_keepalive(struct sock *sk)
939 {
940 	lock_sock(sk);
941 	if (sk->sk_prot->keepalive)
942 		sk->sk_prot->keepalive(sk, true);
943 	sock_valbool_flag(sk, SOCK_KEEPOPEN, true);
944 	release_sock(sk);
945 }
946 EXPORT_SYMBOL(sock_set_keepalive);
947 
__sock_set_rcvbuf(struct sock * sk,int val)948 static void __sock_set_rcvbuf(struct sock *sk, int val)
949 {
950 	/* Ensure val * 2 fits into an int, to prevent max_t() from treating it
951 	 * as a negative value.
952 	 */
953 	val = min_t(int, val, INT_MAX / 2);
954 	sk->sk_userlocks |= SOCK_RCVBUF_LOCK;
955 
956 	/* We double it on the way in to account for "struct sk_buff" etc.
957 	 * overhead.   Applications assume that the SO_RCVBUF setting they make
958 	 * will allow that much actual data to be received on that socket.
959 	 *
960 	 * Applications are unaware that "struct sk_buff" and other overheads
961 	 * allocate from the receive buffer during socket buffer allocation.
962 	 *
963 	 * And after considering the possible alternatives, returning the value
964 	 * we actually used in getsockopt is the most desirable behavior.
965 	 */
966 	WRITE_ONCE(sk->sk_rcvbuf, max_t(int, val * 2, SOCK_MIN_RCVBUF));
967 }
968 
sock_set_rcvbuf(struct sock * sk,int val)969 void sock_set_rcvbuf(struct sock *sk, int val)
970 {
971 	lock_sock(sk);
972 	__sock_set_rcvbuf(sk, val);
973 	release_sock(sk);
974 }
975 EXPORT_SYMBOL(sock_set_rcvbuf);
976 
__sock_set_mark(struct sock * sk,u32 val)977 static void __sock_set_mark(struct sock *sk, u32 val)
978 {
979 	if (val != sk->sk_mark) {
980 		sk->sk_mark = val;
981 		sk_dst_reset(sk);
982 	}
983 }
984 
sock_set_mark(struct sock * sk,u32 val)985 void sock_set_mark(struct sock *sk, u32 val)
986 {
987 	lock_sock(sk);
988 	__sock_set_mark(sk, val);
989 	release_sock(sk);
990 }
991 EXPORT_SYMBOL(sock_set_mark);
992 
sock_release_reserved_memory(struct sock * sk,int bytes)993 static void sock_release_reserved_memory(struct sock *sk, int bytes)
994 {
995 	/* Round down bytes to multiple of pages */
996 	bytes = round_down(bytes, PAGE_SIZE);
997 
998 	WARN_ON(bytes > sk->sk_reserved_mem);
999 	sk->sk_reserved_mem -= bytes;
1000 	sk_mem_reclaim(sk);
1001 }
1002 
sock_reserve_memory(struct sock * sk,int bytes)1003 static int sock_reserve_memory(struct sock *sk, int bytes)
1004 {
1005 	long allocated;
1006 	bool charged;
1007 	int pages;
1008 
1009 	if (!mem_cgroup_sockets_enabled || !sk->sk_memcg || !sk_has_account(sk))
1010 		return -EOPNOTSUPP;
1011 
1012 	if (!bytes)
1013 		return 0;
1014 
1015 	pages = sk_mem_pages(bytes);
1016 
1017 	/* pre-charge to memcg */
1018 	charged = mem_cgroup_charge_skmem(sk->sk_memcg, pages,
1019 					  GFP_KERNEL | __GFP_RETRY_MAYFAIL);
1020 	if (!charged)
1021 		return -ENOMEM;
1022 
1023 	/* pre-charge to forward_alloc */
1024 	sk_memory_allocated_add(sk, pages);
1025 	allocated = sk_memory_allocated(sk);
1026 	/* If the system goes into memory pressure with this
1027 	 * precharge, give up and return error.
1028 	 */
1029 	if (allocated > sk_prot_mem_limits(sk, 1)) {
1030 		sk_memory_allocated_sub(sk, pages);
1031 		mem_cgroup_uncharge_skmem(sk->sk_memcg, pages);
1032 		return -ENOMEM;
1033 	}
1034 	sk->sk_forward_alloc += pages << PAGE_SHIFT;
1035 
1036 	sk->sk_reserved_mem += pages << PAGE_SHIFT;
1037 
1038 	return 0;
1039 }
1040 
sockopt_lock_sock(struct sock * sk)1041 void sockopt_lock_sock(struct sock *sk)
1042 {
1043 	/* When current->bpf_ctx is set, the setsockopt is called from
1044 	 * a bpf prog.  bpf has ensured the sk lock has been
1045 	 * acquired before calling setsockopt().
1046 	 */
1047 	if (has_current_bpf_ctx())
1048 		return;
1049 
1050 	lock_sock(sk);
1051 }
1052 EXPORT_SYMBOL(sockopt_lock_sock);
1053 
sockopt_release_sock(struct sock * sk)1054 void sockopt_release_sock(struct sock *sk)
1055 {
1056 	if (has_current_bpf_ctx())
1057 		return;
1058 
1059 	release_sock(sk);
1060 }
1061 EXPORT_SYMBOL(sockopt_release_sock);
1062 
sockopt_ns_capable(struct user_namespace * ns,int cap)1063 bool sockopt_ns_capable(struct user_namespace *ns, int cap)
1064 {
1065 	return has_current_bpf_ctx() || ns_capable(ns, cap);
1066 }
1067 EXPORT_SYMBOL(sockopt_ns_capable);
1068 
sockopt_capable(int cap)1069 bool sockopt_capable(int cap)
1070 {
1071 	return has_current_bpf_ctx() || capable(cap);
1072 }
1073 EXPORT_SYMBOL(sockopt_capable);
1074 
1075 /*
1076  *	This is meant for all protocols to use and covers goings on
1077  *	at the socket level. Everything here is generic.
1078  */
1079 
sk_setsockopt(struct sock * sk,int level,int optname,sockptr_t optval,unsigned int optlen)1080 int sk_setsockopt(struct sock *sk, int level, int optname,
1081 		  sockptr_t optval, unsigned int optlen)
1082 {
1083 	struct so_timestamping timestamping;
1084 	struct socket *sock = sk->sk_socket;
1085 	struct sock_txtime sk_txtime;
1086 	int val;
1087 	int valbool;
1088 	struct linger ling;
1089 	int ret = 0;
1090 
1091 	/*
1092 	 *	Options without arguments
1093 	 */
1094 
1095 	if (optname == SO_BINDTODEVICE)
1096 		return sock_setbindtodevice(sk, optval, optlen);
1097 
1098 	if (optlen < sizeof(int))
1099 		return -EINVAL;
1100 
1101 	if (copy_from_sockptr(&val, optval, sizeof(val)))
1102 		return -EFAULT;
1103 
1104 	valbool = val ? 1 : 0;
1105 
1106 	sockopt_lock_sock(sk);
1107 
1108 	switch (optname) {
1109 	case SO_DEBUG:
1110 		if (val && !sockopt_capable(CAP_NET_ADMIN))
1111 			ret = -EACCES;
1112 		else
1113 			sock_valbool_flag(sk, SOCK_DBG, valbool);
1114 		break;
1115 	case SO_REUSEADDR:
1116 		sk->sk_reuse = (valbool ? SK_CAN_REUSE : SK_NO_REUSE);
1117 		break;
1118 	case SO_REUSEPORT:
1119 		sk->sk_reuseport = valbool;
1120 		break;
1121 	case SO_TYPE:
1122 	case SO_PROTOCOL:
1123 	case SO_DOMAIN:
1124 	case SO_ERROR:
1125 		ret = -ENOPROTOOPT;
1126 		break;
1127 	case SO_DONTROUTE:
1128 		sock_valbool_flag(sk, SOCK_LOCALROUTE, valbool);
1129 		sk_dst_reset(sk);
1130 		break;
1131 	case SO_BROADCAST:
1132 		sock_valbool_flag(sk, SOCK_BROADCAST, valbool);
1133 		break;
1134 	case SO_SNDBUF:
1135 		/* Don't error on this BSD doesn't and if you think
1136 		 * about it this is right. Otherwise apps have to
1137 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1138 		 * are treated in BSD as hints
1139 		 */
1140 		val = min_t(u32, val, READ_ONCE(sysctl_wmem_max));
1141 set_sndbuf:
1142 		/* Ensure val * 2 fits into an int, to prevent max_t()
1143 		 * from treating it as a negative value.
1144 		 */
1145 		val = min_t(int, val, INT_MAX / 2);
1146 		sk->sk_userlocks |= SOCK_SNDBUF_LOCK;
1147 		WRITE_ONCE(sk->sk_sndbuf,
1148 			   max_t(int, val * 2, SOCK_MIN_SNDBUF));
1149 		/* Wake up sending tasks if we upped the value. */
1150 		sk->sk_write_space(sk);
1151 		break;
1152 
1153 	case SO_SNDBUFFORCE:
1154 		if (!sockopt_capable(CAP_NET_ADMIN)) {
1155 			ret = -EPERM;
1156 			break;
1157 		}
1158 
1159 		/* No negative values (to prevent underflow, as val will be
1160 		 * multiplied by 2).
1161 		 */
1162 		if (val < 0)
1163 			val = 0;
1164 		goto set_sndbuf;
1165 
1166 	case SO_RCVBUF:
1167 		/* Don't error on this BSD doesn't and if you think
1168 		 * about it this is right. Otherwise apps have to
1169 		 * play 'guess the biggest size' games. RCVBUF/SNDBUF
1170 		 * are treated in BSD as hints
1171 		 */
1172 		__sock_set_rcvbuf(sk, min_t(u32, val, READ_ONCE(sysctl_rmem_max)));
1173 		break;
1174 
1175 	case SO_RCVBUFFORCE:
1176 		if (!sockopt_capable(CAP_NET_ADMIN)) {
1177 			ret = -EPERM;
1178 			break;
1179 		}
1180 
1181 		/* No negative values (to prevent underflow, as val will be
1182 		 * multiplied by 2).
1183 		 */
1184 		__sock_set_rcvbuf(sk, max(val, 0));
1185 		break;
1186 
1187 	case SO_KEEPALIVE:
1188 		if (sk->sk_prot->keepalive)
1189 			sk->sk_prot->keepalive(sk, valbool);
1190 		sock_valbool_flag(sk, SOCK_KEEPOPEN, valbool);
1191 		break;
1192 
1193 	case SO_OOBINLINE:
1194 		sock_valbool_flag(sk, SOCK_URGINLINE, valbool);
1195 		break;
1196 
1197 	case SO_NO_CHECK:
1198 		sk->sk_no_check_tx = valbool;
1199 		break;
1200 
1201 	case SO_PRIORITY:
1202 		if ((val >= 0 && val <= 6) ||
1203 		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) ||
1204 		    sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
1205 			sk->sk_priority = val;
1206 		else
1207 			ret = -EPERM;
1208 		break;
1209 
1210 	case SO_LINGER:
1211 		if (optlen < sizeof(ling)) {
1212 			ret = -EINVAL;	/* 1003.1g */
1213 			break;
1214 		}
1215 		if (copy_from_sockptr(&ling, optval, sizeof(ling))) {
1216 			ret = -EFAULT;
1217 			break;
1218 		}
1219 		if (!ling.l_onoff)
1220 			sock_reset_flag(sk, SOCK_LINGER);
1221 		else {
1222 #if (BITS_PER_LONG == 32)
1223 			if ((unsigned int)ling.l_linger >= MAX_SCHEDULE_TIMEOUT/HZ)
1224 				sk->sk_lingertime = MAX_SCHEDULE_TIMEOUT;
1225 			else
1226 #endif
1227 				sk->sk_lingertime = (unsigned int)ling.l_linger * HZ;
1228 			sock_set_flag(sk, SOCK_LINGER);
1229 		}
1230 		break;
1231 
1232 	case SO_BSDCOMPAT:
1233 		break;
1234 
1235 	case SO_PASSCRED:
1236 		if (valbool)
1237 			set_bit(SOCK_PASSCRED, &sock->flags);
1238 		else
1239 			clear_bit(SOCK_PASSCRED, &sock->flags);
1240 		break;
1241 
1242 	case SO_TIMESTAMP_OLD:
1243 	case SO_TIMESTAMP_NEW:
1244 	case SO_TIMESTAMPNS_OLD:
1245 	case SO_TIMESTAMPNS_NEW:
1246 		sock_set_timestamp(sk, optname, valbool);
1247 		break;
1248 
1249 	case SO_TIMESTAMPING_NEW:
1250 	case SO_TIMESTAMPING_OLD:
1251 		if (optlen == sizeof(timestamping)) {
1252 			if (copy_from_sockptr(&timestamping, optval,
1253 					      sizeof(timestamping))) {
1254 				ret = -EFAULT;
1255 				break;
1256 			}
1257 		} else {
1258 			memset(&timestamping, 0, sizeof(timestamping));
1259 			timestamping.flags = val;
1260 		}
1261 		ret = sock_set_timestamping(sk, optname, timestamping);
1262 		break;
1263 
1264 	case SO_RCVLOWAT:
1265 		if (val < 0)
1266 			val = INT_MAX;
1267 		if (sock && sock->ops->set_rcvlowat)
1268 			ret = sock->ops->set_rcvlowat(sk, val);
1269 		else
1270 			WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1271 		break;
1272 
1273 	case SO_RCVTIMEO_OLD:
1274 	case SO_RCVTIMEO_NEW:
1275 		ret = sock_set_timeout(&sk->sk_rcvtimeo, optval,
1276 				       optlen, optname == SO_RCVTIMEO_OLD);
1277 		break;
1278 
1279 	case SO_SNDTIMEO_OLD:
1280 	case SO_SNDTIMEO_NEW:
1281 		ret = sock_set_timeout(&sk->sk_sndtimeo, optval,
1282 				       optlen, optname == SO_SNDTIMEO_OLD);
1283 		break;
1284 
1285 	case SO_ATTACH_FILTER: {
1286 		struct sock_fprog fprog;
1287 
1288 		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1289 		if (!ret)
1290 			ret = sk_attach_filter(&fprog, sk);
1291 		break;
1292 	}
1293 	case SO_ATTACH_BPF:
1294 		ret = -EINVAL;
1295 		if (optlen == sizeof(u32)) {
1296 			u32 ufd;
1297 
1298 			ret = -EFAULT;
1299 			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1300 				break;
1301 
1302 			ret = sk_attach_bpf(ufd, sk);
1303 		}
1304 		break;
1305 
1306 	case SO_ATTACH_REUSEPORT_CBPF: {
1307 		struct sock_fprog fprog;
1308 
1309 		ret = copy_bpf_fprog_from_user(&fprog, optval, optlen);
1310 		if (!ret)
1311 			ret = sk_reuseport_attach_filter(&fprog, sk);
1312 		break;
1313 	}
1314 	case SO_ATTACH_REUSEPORT_EBPF:
1315 		ret = -EINVAL;
1316 		if (optlen == sizeof(u32)) {
1317 			u32 ufd;
1318 
1319 			ret = -EFAULT;
1320 			if (copy_from_sockptr(&ufd, optval, sizeof(ufd)))
1321 				break;
1322 
1323 			ret = sk_reuseport_attach_bpf(ufd, sk);
1324 		}
1325 		break;
1326 
1327 	case SO_DETACH_REUSEPORT_BPF:
1328 		ret = reuseport_detach_prog(sk);
1329 		break;
1330 
1331 	case SO_DETACH_FILTER:
1332 		ret = sk_detach_filter(sk);
1333 		break;
1334 
1335 	case SO_LOCK_FILTER:
1336 		if (sock_flag(sk, SOCK_FILTER_LOCKED) && !valbool)
1337 			ret = -EPERM;
1338 		else
1339 			sock_valbool_flag(sk, SOCK_FILTER_LOCKED, valbool);
1340 		break;
1341 
1342 	case SO_PASSSEC:
1343 		if (valbool)
1344 			set_bit(SOCK_PASSSEC, &sock->flags);
1345 		else
1346 			clear_bit(SOCK_PASSSEC, &sock->flags);
1347 		break;
1348 	case SO_MARK:
1349 		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
1350 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1351 			ret = -EPERM;
1352 			break;
1353 		}
1354 
1355 		__sock_set_mark(sk, val);
1356 		break;
1357 	case SO_RCVMARK:
1358 		if (!sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
1359 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1360 			ret = -EPERM;
1361 			break;
1362 		}
1363 
1364 		sock_valbool_flag(sk, SOCK_RCVMARK, valbool);
1365 		break;
1366 
1367 	case SO_RXQ_OVFL:
1368 		sock_valbool_flag(sk, SOCK_RXQ_OVFL, valbool);
1369 		break;
1370 
1371 	case SO_WIFI_STATUS:
1372 		sock_valbool_flag(sk, SOCK_WIFI_STATUS, valbool);
1373 		break;
1374 
1375 	case SO_PEEK_OFF:
1376 		if (sock->ops->set_peek_off)
1377 			ret = sock->ops->set_peek_off(sk, val);
1378 		else
1379 			ret = -EOPNOTSUPP;
1380 		break;
1381 
1382 	case SO_NOFCS:
1383 		sock_valbool_flag(sk, SOCK_NOFCS, valbool);
1384 		break;
1385 
1386 	case SO_SELECT_ERR_QUEUE:
1387 		sock_valbool_flag(sk, SOCK_SELECT_ERR_QUEUE, valbool);
1388 		break;
1389 
1390 #ifdef CONFIG_NET_RX_BUSY_POLL
1391 	case SO_BUSY_POLL:
1392 		/* allow unprivileged users to decrease the value */
1393 		if ((val > sk->sk_ll_usec) && !sockopt_capable(CAP_NET_ADMIN))
1394 			ret = -EPERM;
1395 		else {
1396 			if (val < 0)
1397 				ret = -EINVAL;
1398 			else
1399 				WRITE_ONCE(sk->sk_ll_usec, val);
1400 		}
1401 		break;
1402 	case SO_PREFER_BUSY_POLL:
1403 		if (valbool && !sockopt_capable(CAP_NET_ADMIN))
1404 			ret = -EPERM;
1405 		else
1406 			WRITE_ONCE(sk->sk_prefer_busy_poll, valbool);
1407 		break;
1408 	case SO_BUSY_POLL_BUDGET:
1409 		if (val > READ_ONCE(sk->sk_busy_poll_budget) && !sockopt_capable(CAP_NET_ADMIN)) {
1410 			ret = -EPERM;
1411 		} else {
1412 			if (val < 0 || val > U16_MAX)
1413 				ret = -EINVAL;
1414 			else
1415 				WRITE_ONCE(sk->sk_busy_poll_budget, val);
1416 		}
1417 		break;
1418 #endif
1419 
1420 	case SO_MAX_PACING_RATE:
1421 		{
1422 		unsigned long ulval = (val == ~0U) ? ~0UL : (unsigned int)val;
1423 
1424 		if (sizeof(ulval) != sizeof(val) &&
1425 		    optlen >= sizeof(ulval) &&
1426 		    copy_from_sockptr(&ulval, optval, sizeof(ulval))) {
1427 			ret = -EFAULT;
1428 			break;
1429 		}
1430 		if (ulval != ~0UL)
1431 			cmpxchg(&sk->sk_pacing_status,
1432 				SK_PACING_NONE,
1433 				SK_PACING_NEEDED);
1434 		sk->sk_max_pacing_rate = ulval;
1435 		sk->sk_pacing_rate = min(sk->sk_pacing_rate, ulval);
1436 		break;
1437 		}
1438 	case SO_INCOMING_CPU:
1439 		reuseport_update_incoming_cpu(sk, val);
1440 		break;
1441 
1442 	case SO_CNX_ADVICE:
1443 		if (val == 1)
1444 			dst_negative_advice(sk);
1445 		break;
1446 
1447 	case SO_ZEROCOPY:
1448 		if (sk->sk_family == PF_INET || sk->sk_family == PF_INET6) {
1449 			if (!(sk_is_tcp(sk) ||
1450 			      (sk->sk_type == SOCK_DGRAM &&
1451 			       sk->sk_protocol == IPPROTO_UDP)))
1452 				ret = -EOPNOTSUPP;
1453 		} else if (sk->sk_family != PF_RDS) {
1454 			ret = -EOPNOTSUPP;
1455 		}
1456 		if (!ret) {
1457 			if (val < 0 || val > 1)
1458 				ret = -EINVAL;
1459 			else
1460 				sock_valbool_flag(sk, SOCK_ZEROCOPY, valbool);
1461 		}
1462 		break;
1463 
1464 	case SO_TXTIME:
1465 		if (optlen != sizeof(struct sock_txtime)) {
1466 			ret = -EINVAL;
1467 			break;
1468 		} else if (copy_from_sockptr(&sk_txtime, optval,
1469 			   sizeof(struct sock_txtime))) {
1470 			ret = -EFAULT;
1471 			break;
1472 		} else if (sk_txtime.flags & ~SOF_TXTIME_FLAGS_MASK) {
1473 			ret = -EINVAL;
1474 			break;
1475 		}
1476 		/* CLOCK_MONOTONIC is only used by sch_fq, and this packet
1477 		 * scheduler has enough safe guards.
1478 		 */
1479 		if (sk_txtime.clockid != CLOCK_MONOTONIC &&
1480 		    !sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {
1481 			ret = -EPERM;
1482 			break;
1483 		}
1484 		sock_valbool_flag(sk, SOCK_TXTIME, true);
1485 		sk->sk_clockid = sk_txtime.clockid;
1486 		sk->sk_txtime_deadline_mode =
1487 			!!(sk_txtime.flags & SOF_TXTIME_DEADLINE_MODE);
1488 		sk->sk_txtime_report_errors =
1489 			!!(sk_txtime.flags & SOF_TXTIME_REPORT_ERRORS);
1490 		break;
1491 
1492 	case SO_BINDTOIFINDEX:
1493 		ret = sock_bindtoindex_locked(sk, val);
1494 		break;
1495 
1496 	case SO_BUF_LOCK:
1497 		if (val & ~SOCK_BUF_LOCK_MASK) {
1498 			ret = -EINVAL;
1499 			break;
1500 		}
1501 		sk->sk_userlocks = val | (sk->sk_userlocks &
1502 					  ~SOCK_BUF_LOCK_MASK);
1503 		break;
1504 
1505 	case SO_RESERVE_MEM:
1506 	{
1507 		int delta;
1508 
1509 		if (val < 0) {
1510 			ret = -EINVAL;
1511 			break;
1512 		}
1513 
1514 		delta = val - sk->sk_reserved_mem;
1515 		if (delta < 0)
1516 			sock_release_reserved_memory(sk, -delta);
1517 		else
1518 			ret = sock_reserve_memory(sk, delta);
1519 		break;
1520 	}
1521 
1522 	case SO_TXREHASH:
1523 		if (val < -1 || val > 1) {
1524 			ret = -EINVAL;
1525 			break;
1526 		}
1527 		/* Paired with READ_ONCE() in tcp_rtx_synack() */
1528 		WRITE_ONCE(sk->sk_txrehash, (u8)val);
1529 		break;
1530 
1531 	default:
1532 		ret = -ENOPROTOOPT;
1533 		break;
1534 	}
1535 	sockopt_release_sock(sk);
1536 	return ret;
1537 }
1538 
sock_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)1539 int sock_setsockopt(struct socket *sock, int level, int optname,
1540 		    sockptr_t optval, unsigned int optlen)
1541 {
1542 	return sk_setsockopt(sock->sk, level, optname,
1543 			     optval, optlen);
1544 }
1545 EXPORT_SYMBOL(sock_setsockopt);
1546 
sk_get_peer_cred(struct sock * sk)1547 static const struct cred *sk_get_peer_cred(struct sock *sk)
1548 {
1549 	const struct cred *cred;
1550 
1551 	spin_lock(&sk->sk_peer_lock);
1552 	cred = get_cred(sk->sk_peer_cred);
1553 	spin_unlock(&sk->sk_peer_lock);
1554 
1555 	return cred;
1556 }
1557 
cred_to_ucred(struct pid * pid,const struct cred * cred,struct ucred * ucred)1558 static void cred_to_ucred(struct pid *pid, const struct cred *cred,
1559 			  struct ucred *ucred)
1560 {
1561 	ucred->pid = pid_vnr(pid);
1562 	ucred->uid = ucred->gid = -1;
1563 	if (cred) {
1564 		struct user_namespace *current_ns = current_user_ns();
1565 
1566 		ucred->uid = from_kuid_munged(current_ns, cred->euid);
1567 		ucred->gid = from_kgid_munged(current_ns, cred->egid);
1568 	}
1569 }
1570 
groups_to_user(sockptr_t dst,const struct group_info * src)1571 static int groups_to_user(sockptr_t dst, const struct group_info *src)
1572 {
1573 	struct user_namespace *user_ns = current_user_ns();
1574 	int i;
1575 
1576 	for (i = 0; i < src->ngroups; i++) {
1577 		gid_t gid = from_kgid_munged(user_ns, src->gid[i]);
1578 
1579 		if (copy_to_sockptr_offset(dst, i * sizeof(gid), &gid, sizeof(gid)))
1580 			return -EFAULT;
1581 	}
1582 
1583 	return 0;
1584 }
1585 
sk_getsockopt(struct sock * sk,int level,int optname,sockptr_t optval,sockptr_t optlen)1586 int sk_getsockopt(struct sock *sk, int level, int optname,
1587 		  sockptr_t optval, sockptr_t optlen)
1588 {
1589 	struct socket *sock = sk->sk_socket;
1590 
1591 	union {
1592 		int val;
1593 		u64 val64;
1594 		unsigned long ulval;
1595 		struct linger ling;
1596 		struct old_timeval32 tm32;
1597 		struct __kernel_old_timeval tm;
1598 		struct  __kernel_sock_timeval stm;
1599 		struct sock_txtime txtime;
1600 		struct so_timestamping timestamping;
1601 	} v;
1602 
1603 	int lv = sizeof(int);
1604 	int len;
1605 
1606 	if (copy_from_sockptr(&len, optlen, sizeof(int)))
1607 		return -EFAULT;
1608 	if (len < 0)
1609 		return -EINVAL;
1610 
1611 	memset(&v, 0, sizeof(v));
1612 
1613 	switch (optname) {
1614 	case SO_DEBUG:
1615 		v.val = sock_flag(sk, SOCK_DBG);
1616 		break;
1617 
1618 	case SO_DONTROUTE:
1619 		v.val = sock_flag(sk, SOCK_LOCALROUTE);
1620 		break;
1621 
1622 	case SO_BROADCAST:
1623 		v.val = sock_flag(sk, SOCK_BROADCAST);
1624 		break;
1625 
1626 	case SO_SNDBUF:
1627 		v.val = sk->sk_sndbuf;
1628 		break;
1629 
1630 	case SO_RCVBUF:
1631 		v.val = sk->sk_rcvbuf;
1632 		break;
1633 
1634 	case SO_REUSEADDR:
1635 		v.val = sk->sk_reuse;
1636 		break;
1637 
1638 	case SO_REUSEPORT:
1639 		v.val = sk->sk_reuseport;
1640 		break;
1641 
1642 	case SO_KEEPALIVE:
1643 		v.val = sock_flag(sk, SOCK_KEEPOPEN);
1644 		break;
1645 
1646 	case SO_TYPE:
1647 		v.val = sk->sk_type;
1648 		break;
1649 
1650 	case SO_PROTOCOL:
1651 		v.val = sk->sk_protocol;
1652 		break;
1653 
1654 	case SO_DOMAIN:
1655 		v.val = sk->sk_family;
1656 		break;
1657 
1658 	case SO_ERROR:
1659 		v.val = -sock_error(sk);
1660 		if (v.val == 0)
1661 			v.val = xchg(&sk->sk_err_soft, 0);
1662 		break;
1663 
1664 	case SO_OOBINLINE:
1665 		v.val = sock_flag(sk, SOCK_URGINLINE);
1666 		break;
1667 
1668 	case SO_NO_CHECK:
1669 		v.val = sk->sk_no_check_tx;
1670 		break;
1671 
1672 	case SO_PRIORITY:
1673 		v.val = sk->sk_priority;
1674 		break;
1675 
1676 	case SO_LINGER:
1677 		lv		= sizeof(v.ling);
1678 		v.ling.l_onoff	= sock_flag(sk, SOCK_LINGER);
1679 		v.ling.l_linger	= sk->sk_lingertime / HZ;
1680 		break;
1681 
1682 	case SO_BSDCOMPAT:
1683 		break;
1684 
1685 	case SO_TIMESTAMP_OLD:
1686 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) &&
1687 				!sock_flag(sk, SOCK_TSTAMP_NEW) &&
1688 				!sock_flag(sk, SOCK_RCVTSTAMPNS);
1689 		break;
1690 
1691 	case SO_TIMESTAMPNS_OLD:
1692 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && !sock_flag(sk, SOCK_TSTAMP_NEW);
1693 		break;
1694 
1695 	case SO_TIMESTAMP_NEW:
1696 		v.val = sock_flag(sk, SOCK_RCVTSTAMP) && sock_flag(sk, SOCK_TSTAMP_NEW);
1697 		break;
1698 
1699 	case SO_TIMESTAMPNS_NEW:
1700 		v.val = sock_flag(sk, SOCK_RCVTSTAMPNS) && sock_flag(sk, SOCK_TSTAMP_NEW);
1701 		break;
1702 
1703 	case SO_TIMESTAMPING_OLD:
1704 		lv = sizeof(v.timestamping);
1705 		v.timestamping.flags = sk->sk_tsflags;
1706 		v.timestamping.bind_phc = sk->sk_bind_phc;
1707 		break;
1708 
1709 	case SO_RCVTIMEO_OLD:
1710 	case SO_RCVTIMEO_NEW:
1711 		lv = sock_get_timeout(sk->sk_rcvtimeo, &v, SO_RCVTIMEO_OLD == optname);
1712 		break;
1713 
1714 	case SO_SNDTIMEO_OLD:
1715 	case SO_SNDTIMEO_NEW:
1716 		lv = sock_get_timeout(sk->sk_sndtimeo, &v, SO_SNDTIMEO_OLD == optname);
1717 		break;
1718 
1719 	case SO_RCVLOWAT:
1720 		v.val = sk->sk_rcvlowat;
1721 		break;
1722 
1723 	case SO_SNDLOWAT:
1724 		v.val = 1;
1725 		break;
1726 
1727 	case SO_PASSCRED:
1728 		v.val = !!test_bit(SOCK_PASSCRED, &sock->flags);
1729 		break;
1730 
1731 	case SO_PEERCRED:
1732 	{
1733 		struct ucred peercred;
1734 		if (len > sizeof(peercred))
1735 			len = sizeof(peercred);
1736 
1737 		spin_lock(&sk->sk_peer_lock);
1738 		cred_to_ucred(sk->sk_peer_pid, sk->sk_peer_cred, &peercred);
1739 		spin_unlock(&sk->sk_peer_lock);
1740 
1741 		if (copy_to_sockptr(optval, &peercred, len))
1742 			return -EFAULT;
1743 		goto lenout;
1744 	}
1745 
1746 	case SO_PEERGROUPS:
1747 	{
1748 		const struct cred *cred;
1749 		int ret, n;
1750 
1751 		cred = sk_get_peer_cred(sk);
1752 		if (!cred)
1753 			return -ENODATA;
1754 
1755 		n = cred->group_info->ngroups;
1756 		if (len < n * sizeof(gid_t)) {
1757 			len = n * sizeof(gid_t);
1758 			put_cred(cred);
1759 			return copy_to_sockptr(optlen, &len, sizeof(int)) ? -EFAULT : -ERANGE;
1760 		}
1761 		len = n * sizeof(gid_t);
1762 
1763 		ret = groups_to_user(optval, cred->group_info);
1764 		put_cred(cred);
1765 		if (ret)
1766 			return ret;
1767 		goto lenout;
1768 	}
1769 
1770 	case SO_PEERNAME:
1771 	{
1772 		char address[128];
1773 
1774 		lv = sock->ops->getname(sock, (struct sockaddr *)address, 2);
1775 		if (lv < 0)
1776 			return -ENOTCONN;
1777 		if (lv < len)
1778 			return -EINVAL;
1779 		if (copy_to_sockptr(optval, address, len))
1780 			return -EFAULT;
1781 		goto lenout;
1782 	}
1783 
1784 	/* Dubious BSD thing... Probably nobody even uses it, but
1785 	 * the UNIX standard wants it for whatever reason... -DaveM
1786 	 */
1787 	case SO_ACCEPTCONN:
1788 		v.val = sk->sk_state == TCP_LISTEN;
1789 		break;
1790 
1791 	case SO_PASSSEC:
1792 		v.val = !!test_bit(SOCK_PASSSEC, &sock->flags);
1793 		break;
1794 
1795 	case SO_PEERSEC:
1796 		return security_socket_getpeersec_stream(sock, optval.user, optlen.user, len);
1797 
1798 	case SO_MARK:
1799 		v.val = sk->sk_mark;
1800 		break;
1801 
1802 	case SO_RCVMARK:
1803 		v.val = sock_flag(sk, SOCK_RCVMARK);
1804 		break;
1805 
1806 	case SO_RXQ_OVFL:
1807 		v.val = sock_flag(sk, SOCK_RXQ_OVFL);
1808 		break;
1809 
1810 	case SO_WIFI_STATUS:
1811 		v.val = sock_flag(sk, SOCK_WIFI_STATUS);
1812 		break;
1813 
1814 	case SO_PEEK_OFF:
1815 		if (!sock->ops->set_peek_off)
1816 			return -EOPNOTSUPP;
1817 
1818 		v.val = sk->sk_peek_off;
1819 		break;
1820 	case SO_NOFCS:
1821 		v.val = sock_flag(sk, SOCK_NOFCS);
1822 		break;
1823 
1824 	case SO_BINDTODEVICE:
1825 		return sock_getbindtodevice(sk, optval, optlen, len);
1826 
1827 	case SO_GET_FILTER:
1828 		len = sk_get_filter(sk, optval, len);
1829 		if (len < 0)
1830 			return len;
1831 
1832 		goto lenout;
1833 
1834 	case SO_LOCK_FILTER:
1835 		v.val = sock_flag(sk, SOCK_FILTER_LOCKED);
1836 		break;
1837 
1838 	case SO_BPF_EXTENSIONS:
1839 		v.val = bpf_tell_extensions();
1840 		break;
1841 
1842 	case SO_SELECT_ERR_QUEUE:
1843 		v.val = sock_flag(sk, SOCK_SELECT_ERR_QUEUE);
1844 		break;
1845 
1846 #ifdef CONFIG_NET_RX_BUSY_POLL
1847 	case SO_BUSY_POLL:
1848 		v.val = sk->sk_ll_usec;
1849 		break;
1850 	case SO_PREFER_BUSY_POLL:
1851 		v.val = READ_ONCE(sk->sk_prefer_busy_poll);
1852 		break;
1853 #endif
1854 
1855 	case SO_MAX_PACING_RATE:
1856 		if (sizeof(v.ulval) != sizeof(v.val) && len >= sizeof(v.ulval)) {
1857 			lv = sizeof(v.ulval);
1858 			v.ulval = sk->sk_max_pacing_rate;
1859 		} else {
1860 			/* 32bit version */
1861 			v.val = min_t(unsigned long, sk->sk_max_pacing_rate, ~0U);
1862 		}
1863 		break;
1864 
1865 	case SO_INCOMING_CPU:
1866 		v.val = READ_ONCE(sk->sk_incoming_cpu);
1867 		break;
1868 
1869 	case SO_MEMINFO:
1870 	{
1871 		u32 meminfo[SK_MEMINFO_VARS];
1872 
1873 		sk_get_meminfo(sk, meminfo);
1874 
1875 		len = min_t(unsigned int, len, sizeof(meminfo));
1876 		if (copy_to_sockptr(optval, &meminfo, len))
1877 			return -EFAULT;
1878 
1879 		goto lenout;
1880 	}
1881 
1882 #ifdef CONFIG_NET_RX_BUSY_POLL
1883 	case SO_INCOMING_NAPI_ID:
1884 		v.val = READ_ONCE(sk->sk_napi_id);
1885 
1886 		/* aggregate non-NAPI IDs down to 0 */
1887 		if (v.val < MIN_NAPI_ID)
1888 			v.val = 0;
1889 
1890 		break;
1891 #endif
1892 
1893 	case SO_COOKIE:
1894 		lv = sizeof(u64);
1895 		if (len < lv)
1896 			return -EINVAL;
1897 		v.val64 = sock_gen_cookie(sk);
1898 		break;
1899 
1900 	case SO_ZEROCOPY:
1901 		v.val = sock_flag(sk, SOCK_ZEROCOPY);
1902 		break;
1903 
1904 	case SO_TXTIME:
1905 		lv = sizeof(v.txtime);
1906 		v.txtime.clockid = sk->sk_clockid;
1907 		v.txtime.flags |= sk->sk_txtime_deadline_mode ?
1908 				  SOF_TXTIME_DEADLINE_MODE : 0;
1909 		v.txtime.flags |= sk->sk_txtime_report_errors ?
1910 				  SOF_TXTIME_REPORT_ERRORS : 0;
1911 		break;
1912 
1913 	case SO_BINDTOIFINDEX:
1914 		v.val = READ_ONCE(sk->sk_bound_dev_if);
1915 		break;
1916 
1917 	case SO_NETNS_COOKIE:
1918 		lv = sizeof(u64);
1919 		if (len != lv)
1920 			return -EINVAL;
1921 		v.val64 = sock_net(sk)->net_cookie;
1922 		break;
1923 
1924 	case SO_BUF_LOCK:
1925 		v.val = sk->sk_userlocks & SOCK_BUF_LOCK_MASK;
1926 		break;
1927 
1928 	case SO_RESERVE_MEM:
1929 		v.val = sk->sk_reserved_mem;
1930 		break;
1931 
1932 	case SO_TXREHASH:
1933 		v.val = sk->sk_txrehash;
1934 		break;
1935 
1936 	default:
1937 		/* We implement the SO_SNDLOWAT etc to not be settable
1938 		 * (1003.1g 7).
1939 		 */
1940 		return -ENOPROTOOPT;
1941 	}
1942 
1943 	if (len > lv)
1944 		len = lv;
1945 	if (copy_to_sockptr(optval, &v, len))
1946 		return -EFAULT;
1947 lenout:
1948 	if (copy_to_sockptr(optlen, &len, sizeof(int)))
1949 		return -EFAULT;
1950 	return 0;
1951 }
1952 
sock_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)1953 int sock_getsockopt(struct socket *sock, int level, int optname,
1954 		    char __user *optval, int __user *optlen)
1955 {
1956 	return sk_getsockopt(sock->sk, level, optname,
1957 			     USER_SOCKPTR(optval),
1958 			     USER_SOCKPTR(optlen));
1959 }
1960 
1961 /*
1962  * Initialize an sk_lock.
1963  *
1964  * (We also register the sk_lock with the lock validator.)
1965  */
sock_lock_init(struct sock * sk)1966 static inline void sock_lock_init(struct sock *sk)
1967 {
1968 	if (sk->sk_kern_sock)
1969 		sock_lock_init_class_and_name(
1970 			sk,
1971 			af_family_kern_slock_key_strings[sk->sk_family],
1972 			af_family_kern_slock_keys + sk->sk_family,
1973 			af_family_kern_key_strings[sk->sk_family],
1974 			af_family_kern_keys + sk->sk_family);
1975 	else
1976 		sock_lock_init_class_and_name(
1977 			sk,
1978 			af_family_slock_key_strings[sk->sk_family],
1979 			af_family_slock_keys + sk->sk_family,
1980 			af_family_key_strings[sk->sk_family],
1981 			af_family_keys + sk->sk_family);
1982 }
1983 
1984 /*
1985  * Copy all fields from osk to nsk but nsk->sk_refcnt must not change yet,
1986  * even temporarly, because of RCU lookups. sk_node should also be left as is.
1987  * We must not copy fields between sk_dontcopy_begin and sk_dontcopy_end
1988  */
sock_copy(struct sock * nsk,const struct sock * osk)1989 static void sock_copy(struct sock *nsk, const struct sock *osk)
1990 {
1991 	const struct proto *prot = READ_ONCE(osk->sk_prot);
1992 #ifdef CONFIG_SECURITY_NETWORK
1993 	void *sptr = nsk->sk_security;
1994 #endif
1995 
1996 	/* If we move sk_tx_queue_mapping out of the private section,
1997 	 * we must check if sk_tx_queue_clear() is called after
1998 	 * sock_copy() in sk_clone_lock().
1999 	 */
2000 	BUILD_BUG_ON(offsetof(struct sock, sk_tx_queue_mapping) <
2001 		     offsetof(struct sock, sk_dontcopy_begin) ||
2002 		     offsetof(struct sock, sk_tx_queue_mapping) >=
2003 		     offsetof(struct sock, sk_dontcopy_end));
2004 
2005 	memcpy(nsk, osk, offsetof(struct sock, sk_dontcopy_begin));
2006 
2007 	memcpy(&nsk->sk_dontcopy_end, &osk->sk_dontcopy_end,
2008 	       prot->obj_size - offsetof(struct sock, sk_dontcopy_end));
2009 
2010 #ifdef CONFIG_SECURITY_NETWORK
2011 	nsk->sk_security = sptr;
2012 	security_sk_clone(osk, nsk);
2013 #endif
2014 }
2015 
sk_prot_alloc(struct proto * prot,gfp_t priority,int family)2016 static struct sock *sk_prot_alloc(struct proto *prot, gfp_t priority,
2017 		int family)
2018 {
2019 	struct sock *sk;
2020 	struct kmem_cache *slab;
2021 
2022 	slab = prot->slab;
2023 	if (slab != NULL) {
2024 		sk = kmem_cache_alloc(slab, priority & ~__GFP_ZERO);
2025 		if (!sk)
2026 			return sk;
2027 		if (want_init_on_alloc(priority))
2028 			sk_prot_clear_nulls(sk, prot->obj_size);
2029 	} else
2030 		sk = kmalloc(prot->obj_size, priority);
2031 
2032 	if (sk != NULL) {
2033 		if (security_sk_alloc(sk, family, priority))
2034 			goto out_free;
2035 
2036 		if (!try_module_get(prot->owner))
2037 			goto out_free_sec;
2038 	}
2039 
2040 	return sk;
2041 
2042 out_free_sec:
2043 	security_sk_free(sk);
2044 out_free:
2045 	if (slab != NULL)
2046 		kmem_cache_free(slab, sk);
2047 	else
2048 		kfree(sk);
2049 	return NULL;
2050 }
2051 
sk_prot_free(struct proto * prot,struct sock * sk)2052 static void sk_prot_free(struct proto *prot, struct sock *sk)
2053 {
2054 	struct kmem_cache *slab;
2055 	struct module *owner;
2056 
2057 	owner = prot->owner;
2058 	slab = prot->slab;
2059 
2060 	cgroup_sk_free(&sk->sk_cgrp_data);
2061 	mem_cgroup_sk_free(sk);
2062 	security_sk_free(sk);
2063 	if (slab != NULL)
2064 		kmem_cache_free(slab, sk);
2065 	else
2066 		kfree(sk);
2067 	module_put(owner);
2068 }
2069 
2070 /**
2071  *	sk_alloc - All socket objects are allocated here
2072  *	@net: the applicable net namespace
2073  *	@family: protocol family
2074  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2075  *	@prot: struct proto associated with this new sock instance
2076  *	@kern: is this to be a kernel socket?
2077  */
sk_alloc(struct net * net,int family,gfp_t priority,struct proto * prot,int kern)2078 struct sock *sk_alloc(struct net *net, int family, gfp_t priority,
2079 		      struct proto *prot, int kern)
2080 {
2081 	struct sock *sk;
2082 
2083 	sk = sk_prot_alloc(prot, priority | __GFP_ZERO, family);
2084 	if (sk) {
2085 		sk->sk_family = family;
2086 		/*
2087 		 * See comment in struct sock definition to understand
2088 		 * why we need sk_prot_creator -acme
2089 		 */
2090 		sk->sk_prot = sk->sk_prot_creator = prot;
2091 		sk->sk_kern_sock = kern;
2092 		sock_lock_init(sk);
2093 		sk->sk_net_refcnt = kern ? 0 : 1;
2094 		if (likely(sk->sk_net_refcnt)) {
2095 			get_net_track(net, &sk->ns_tracker, priority);
2096 			sock_inuse_add(net, 1);
2097 		}
2098 
2099 		sock_net_set(sk, net);
2100 		refcount_set(&sk->sk_wmem_alloc, 1);
2101 
2102 		mem_cgroup_sk_alloc(sk);
2103 		cgroup_sk_alloc(&sk->sk_cgrp_data);
2104 		sock_update_classid(&sk->sk_cgrp_data);
2105 		sock_update_netprioidx(&sk->sk_cgrp_data);
2106 		sk_tx_queue_clear(sk);
2107 	}
2108 
2109 	return sk;
2110 }
2111 EXPORT_SYMBOL(sk_alloc);
2112 
2113 /* Sockets having SOCK_RCU_FREE will call this function after one RCU
2114  * grace period. This is the case for UDP sockets and TCP listeners.
2115  */
__sk_destruct(struct rcu_head * head)2116 static void __sk_destruct(struct rcu_head *head)
2117 {
2118 	struct sock *sk = container_of(head, struct sock, sk_rcu);
2119 	struct sk_filter *filter;
2120 
2121 	if (sk->sk_destruct)
2122 		sk->sk_destruct(sk);
2123 
2124 	filter = rcu_dereference_check(sk->sk_filter,
2125 				       refcount_read(&sk->sk_wmem_alloc) == 0);
2126 	if (filter) {
2127 		sk_filter_uncharge(sk, filter);
2128 		RCU_INIT_POINTER(sk->sk_filter, NULL);
2129 	}
2130 
2131 	sock_disable_timestamp(sk, SK_FLAGS_TIMESTAMP);
2132 
2133 #ifdef CONFIG_BPF_SYSCALL
2134 	bpf_sk_storage_free(sk);
2135 #endif
2136 
2137 	if (atomic_read(&sk->sk_omem_alloc))
2138 		pr_debug("%s: optmem leakage (%d bytes) detected\n",
2139 			 __func__, atomic_read(&sk->sk_omem_alloc));
2140 
2141 	if (sk->sk_frag.page) {
2142 		put_page(sk->sk_frag.page);
2143 		sk->sk_frag.page = NULL;
2144 	}
2145 
2146 	/* We do not need to acquire sk->sk_peer_lock, we are the last user. */
2147 	put_cred(sk->sk_peer_cred);
2148 	put_pid(sk->sk_peer_pid);
2149 
2150 	if (likely(sk->sk_net_refcnt))
2151 		put_net_track(sock_net(sk), &sk->ns_tracker);
2152 	sk_prot_free(sk->sk_prot_creator, sk);
2153 }
2154 
sk_destruct(struct sock * sk)2155 void sk_destruct(struct sock *sk)
2156 {
2157 	bool use_call_rcu = sock_flag(sk, SOCK_RCU_FREE);
2158 
2159 	if (rcu_access_pointer(sk->sk_reuseport_cb)) {
2160 		reuseport_detach_sock(sk);
2161 		use_call_rcu = true;
2162 	}
2163 
2164 	if (use_call_rcu)
2165 		call_rcu(&sk->sk_rcu, __sk_destruct);
2166 	else
2167 		__sk_destruct(&sk->sk_rcu);
2168 }
2169 
__sk_free(struct sock * sk)2170 static void __sk_free(struct sock *sk)
2171 {
2172 	if (likely(sk->sk_net_refcnt))
2173 		sock_inuse_add(sock_net(sk), -1);
2174 
2175 	if (unlikely(sk->sk_net_refcnt && sock_diag_has_destroy_listeners(sk)))
2176 		sock_diag_broadcast_destroy(sk);
2177 	else
2178 		sk_destruct(sk);
2179 }
2180 
sk_free(struct sock * sk)2181 void sk_free(struct sock *sk)
2182 {
2183 	/*
2184 	 * We subtract one from sk_wmem_alloc and can know if
2185 	 * some packets are still in some tx queue.
2186 	 * If not null, sock_wfree() will call __sk_free(sk) later
2187 	 */
2188 	if (refcount_dec_and_test(&sk->sk_wmem_alloc))
2189 		__sk_free(sk);
2190 }
2191 EXPORT_SYMBOL(sk_free);
2192 
sk_init_common(struct sock * sk)2193 static void sk_init_common(struct sock *sk)
2194 {
2195 	skb_queue_head_init(&sk->sk_receive_queue);
2196 	skb_queue_head_init(&sk->sk_write_queue);
2197 	skb_queue_head_init(&sk->sk_error_queue);
2198 
2199 	rwlock_init(&sk->sk_callback_lock);
2200 	lockdep_set_class_and_name(&sk->sk_receive_queue.lock,
2201 			af_rlock_keys + sk->sk_family,
2202 			af_family_rlock_key_strings[sk->sk_family]);
2203 	lockdep_set_class_and_name(&sk->sk_write_queue.lock,
2204 			af_wlock_keys + sk->sk_family,
2205 			af_family_wlock_key_strings[sk->sk_family]);
2206 	lockdep_set_class_and_name(&sk->sk_error_queue.lock,
2207 			af_elock_keys + sk->sk_family,
2208 			af_family_elock_key_strings[sk->sk_family]);
2209 	lockdep_set_class_and_name(&sk->sk_callback_lock,
2210 			af_callback_keys + sk->sk_family,
2211 			af_family_clock_key_strings[sk->sk_family]);
2212 }
2213 
2214 /**
2215  *	sk_clone_lock - clone a socket, and lock its clone
2216  *	@sk: the socket to clone
2217  *	@priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
2218  *
2219  *	Caller must unlock socket even in error path (bh_unlock_sock(newsk))
2220  */
sk_clone_lock(const struct sock * sk,const gfp_t priority)2221 struct sock *sk_clone_lock(const struct sock *sk, const gfp_t priority)
2222 {
2223 	struct proto *prot = READ_ONCE(sk->sk_prot);
2224 	struct sk_filter *filter;
2225 	bool is_charged = true;
2226 	struct sock *newsk;
2227 
2228 	newsk = sk_prot_alloc(prot, priority, sk->sk_family);
2229 	if (!newsk)
2230 		goto out;
2231 
2232 	sock_copy(newsk, sk);
2233 
2234 	newsk->sk_prot_creator = prot;
2235 
2236 	/* SANITY */
2237 	if (likely(newsk->sk_net_refcnt)) {
2238 		get_net_track(sock_net(newsk), &newsk->ns_tracker, priority);
2239 		sock_inuse_add(sock_net(newsk), 1);
2240 	}
2241 	sk_node_init(&newsk->sk_node);
2242 	sock_lock_init(newsk);
2243 	bh_lock_sock(newsk);
2244 	newsk->sk_backlog.head	= newsk->sk_backlog.tail = NULL;
2245 	newsk->sk_backlog.len = 0;
2246 
2247 	atomic_set(&newsk->sk_rmem_alloc, 0);
2248 
2249 	/* sk_wmem_alloc set to one (see sk_free() and sock_wfree()) */
2250 	refcount_set(&newsk->sk_wmem_alloc, 1);
2251 
2252 	atomic_set(&newsk->sk_omem_alloc, 0);
2253 	sk_init_common(newsk);
2254 
2255 	newsk->sk_dst_cache	= NULL;
2256 	newsk->sk_dst_pending_confirm = 0;
2257 	newsk->sk_wmem_queued	= 0;
2258 	newsk->sk_forward_alloc = 0;
2259 	newsk->sk_reserved_mem  = 0;
2260 	atomic_set(&newsk->sk_drops, 0);
2261 	newsk->sk_send_head	= NULL;
2262 	newsk->sk_userlocks	= sk->sk_userlocks & ~SOCK_BINDPORT_LOCK;
2263 	atomic_set(&newsk->sk_zckey, 0);
2264 
2265 	sock_reset_flag(newsk, SOCK_DONE);
2266 
2267 	/* sk->sk_memcg will be populated at accept() time */
2268 	newsk->sk_memcg = NULL;
2269 
2270 	cgroup_sk_clone(&newsk->sk_cgrp_data);
2271 
2272 	rcu_read_lock();
2273 	filter = rcu_dereference(sk->sk_filter);
2274 	if (filter != NULL)
2275 		/* though it's an empty new sock, the charging may fail
2276 		 * if sysctl_optmem_max was changed between creation of
2277 		 * original socket and cloning
2278 		 */
2279 		is_charged = sk_filter_charge(newsk, filter);
2280 	RCU_INIT_POINTER(newsk->sk_filter, filter);
2281 	rcu_read_unlock();
2282 
2283 	if (unlikely(!is_charged || xfrm_sk_clone_policy(newsk, sk))) {
2284 		/* We need to make sure that we don't uncharge the new
2285 		 * socket if we couldn't charge it in the first place
2286 		 * as otherwise we uncharge the parent's filter.
2287 		 */
2288 		if (!is_charged)
2289 			RCU_INIT_POINTER(newsk->sk_filter, NULL);
2290 		sk_free_unlock_clone(newsk);
2291 		newsk = NULL;
2292 		goto out;
2293 	}
2294 	RCU_INIT_POINTER(newsk->sk_reuseport_cb, NULL);
2295 
2296 	if (bpf_sk_storage_clone(sk, newsk)) {
2297 		sk_free_unlock_clone(newsk);
2298 		newsk = NULL;
2299 		goto out;
2300 	}
2301 
2302 	/* Clear sk_user_data if parent had the pointer tagged
2303 	 * as not suitable for copying when cloning.
2304 	 */
2305 	if (sk_user_data_is_nocopy(newsk))
2306 		newsk->sk_user_data = NULL;
2307 
2308 	newsk->sk_err	   = 0;
2309 	newsk->sk_err_soft = 0;
2310 	newsk->sk_priority = 0;
2311 	newsk->sk_incoming_cpu = raw_smp_processor_id();
2312 
2313 	/* Before updating sk_refcnt, we must commit prior changes to memory
2314 	 * (Documentation/RCU/rculist_nulls.rst for details)
2315 	 */
2316 	smp_wmb();
2317 	refcount_set(&newsk->sk_refcnt, 2);
2318 
2319 	/* Increment the counter in the same struct proto as the master
2320 	 * sock (sk_refcnt_debug_inc uses newsk->sk_prot->socks, that
2321 	 * is the same as sk->sk_prot->socks, as this field was copied
2322 	 * with memcpy).
2323 	 *
2324 	 * This _changes_ the previous behaviour, where
2325 	 * tcp_create_openreq_child always was incrementing the
2326 	 * equivalent to tcp_prot->socks (inet_sock_nr), so this have
2327 	 * to be taken into account in all callers. -acme
2328 	 */
2329 	sk_refcnt_debug_inc(newsk);
2330 	sk_set_socket(newsk, NULL);
2331 	sk_tx_queue_clear(newsk);
2332 	RCU_INIT_POINTER(newsk->sk_wq, NULL);
2333 
2334 	if (newsk->sk_prot->sockets_allocated)
2335 		sk_sockets_allocated_inc(newsk);
2336 
2337 	if (sock_needs_netstamp(sk) && newsk->sk_flags & SK_FLAGS_TIMESTAMP)
2338 		net_enable_timestamp();
2339 out:
2340 	return newsk;
2341 }
2342 EXPORT_SYMBOL_GPL(sk_clone_lock);
2343 
sk_free_unlock_clone(struct sock * sk)2344 void sk_free_unlock_clone(struct sock *sk)
2345 {
2346 	/* It is still raw copy of parent, so invalidate
2347 	 * destructor and make plain sk_free() */
2348 	sk->sk_destruct = NULL;
2349 	bh_unlock_sock(sk);
2350 	sk_free(sk);
2351 }
2352 EXPORT_SYMBOL_GPL(sk_free_unlock_clone);
2353 
sk_trim_gso_size(struct sock * sk)2354 static void sk_trim_gso_size(struct sock *sk)
2355 {
2356 	if (sk->sk_gso_max_size <= GSO_LEGACY_MAX_SIZE)
2357 		return;
2358 #if IS_ENABLED(CONFIG_IPV6)
2359 	if (sk->sk_family == AF_INET6 &&
2360 	    sk_is_tcp(sk) &&
2361 	    !ipv6_addr_v4mapped(&sk->sk_v6_rcv_saddr))
2362 		return;
2363 #endif
2364 	sk->sk_gso_max_size = GSO_LEGACY_MAX_SIZE;
2365 }
2366 
sk_setup_caps(struct sock * sk,struct dst_entry * dst)2367 void sk_setup_caps(struct sock *sk, struct dst_entry *dst)
2368 {
2369 	u32 max_segs = 1;
2370 
2371 	sk_dst_set(sk, dst);
2372 	sk->sk_route_caps = dst->dev->features;
2373 	if (sk_is_tcp(sk))
2374 		sk->sk_route_caps |= NETIF_F_GSO;
2375 	if (sk->sk_route_caps & NETIF_F_GSO)
2376 		sk->sk_route_caps |= NETIF_F_GSO_SOFTWARE;
2377 	if (unlikely(sk->sk_gso_disabled))
2378 		sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2379 	if (sk_can_gso(sk)) {
2380 		if (dst->header_len && !xfrm_dst_offload_ok(dst)) {
2381 			sk->sk_route_caps &= ~NETIF_F_GSO_MASK;
2382 		} else {
2383 			sk->sk_route_caps |= NETIF_F_SG | NETIF_F_HW_CSUM;
2384 			/* pairs with the WRITE_ONCE() in netif_set_gso_max_size() */
2385 			sk->sk_gso_max_size = READ_ONCE(dst->dev->gso_max_size);
2386 			sk_trim_gso_size(sk);
2387 			sk->sk_gso_max_size -= (MAX_TCP_HEADER + 1);
2388 			/* pairs with the WRITE_ONCE() in netif_set_gso_max_segs() */
2389 			max_segs = max_t(u32, READ_ONCE(dst->dev->gso_max_segs), 1);
2390 		}
2391 	}
2392 	sk->sk_gso_max_segs = max_segs;
2393 }
2394 EXPORT_SYMBOL_GPL(sk_setup_caps);
2395 
2396 /*
2397  *	Simple resource managers for sockets.
2398  */
2399 
2400 
2401 /*
2402  * Write buffer destructor automatically called from kfree_skb.
2403  */
sock_wfree(struct sk_buff * skb)2404 void sock_wfree(struct sk_buff *skb)
2405 {
2406 	struct sock *sk = skb->sk;
2407 	unsigned int len = skb->truesize;
2408 	bool free;
2409 
2410 	if (!sock_flag(sk, SOCK_USE_WRITE_QUEUE)) {
2411 		if (sock_flag(sk, SOCK_RCU_FREE) &&
2412 		    sk->sk_write_space == sock_def_write_space) {
2413 			rcu_read_lock();
2414 			free = refcount_sub_and_test(len, &sk->sk_wmem_alloc);
2415 			sock_def_write_space_wfree(sk);
2416 			rcu_read_unlock();
2417 			if (unlikely(free))
2418 				__sk_free(sk);
2419 			return;
2420 		}
2421 
2422 		/*
2423 		 * Keep a reference on sk_wmem_alloc, this will be released
2424 		 * after sk_write_space() call
2425 		 */
2426 		WARN_ON(refcount_sub_and_test(len - 1, &sk->sk_wmem_alloc));
2427 		sk->sk_write_space(sk);
2428 		len = 1;
2429 	}
2430 	/*
2431 	 * if sk_wmem_alloc reaches 0, we must finish what sk_free()
2432 	 * could not do because of in-flight packets
2433 	 */
2434 	if (refcount_sub_and_test(len, &sk->sk_wmem_alloc))
2435 		__sk_free(sk);
2436 }
2437 EXPORT_SYMBOL(sock_wfree);
2438 
2439 /* This variant of sock_wfree() is used by TCP,
2440  * since it sets SOCK_USE_WRITE_QUEUE.
2441  */
__sock_wfree(struct sk_buff * skb)2442 void __sock_wfree(struct sk_buff *skb)
2443 {
2444 	struct sock *sk = skb->sk;
2445 
2446 	if (refcount_sub_and_test(skb->truesize, &sk->sk_wmem_alloc))
2447 		__sk_free(sk);
2448 }
2449 
skb_set_owner_w(struct sk_buff * skb,struct sock * sk)2450 void skb_set_owner_w(struct sk_buff *skb, struct sock *sk)
2451 {
2452 	skb_orphan(skb);
2453 	skb->sk = sk;
2454 #ifdef CONFIG_INET
2455 	if (unlikely(!sk_fullsock(sk))) {
2456 		skb->destructor = sock_edemux;
2457 		sock_hold(sk);
2458 		return;
2459 	}
2460 #endif
2461 	skb->destructor = sock_wfree;
2462 	skb_set_hash_from_sk(skb, sk);
2463 	/*
2464 	 * We used to take a refcount on sk, but following operation
2465 	 * is enough to guarantee sk_free() wont free this sock until
2466 	 * all in-flight packets are completed
2467 	 */
2468 	refcount_add(skb->truesize, &sk->sk_wmem_alloc);
2469 }
2470 EXPORT_SYMBOL(skb_set_owner_w);
2471 
can_skb_orphan_partial(const struct sk_buff * skb)2472 static bool can_skb_orphan_partial(const struct sk_buff *skb)
2473 {
2474 #ifdef CONFIG_TLS_DEVICE
2475 	/* Drivers depend on in-order delivery for crypto offload,
2476 	 * partial orphan breaks out-of-order-OK logic.
2477 	 */
2478 	if (skb->decrypted)
2479 		return false;
2480 #endif
2481 	return (skb->destructor == sock_wfree ||
2482 		(IS_ENABLED(CONFIG_INET) && skb->destructor == tcp_wfree));
2483 }
2484 
2485 /* This helper is used by netem, as it can hold packets in its
2486  * delay queue. We want to allow the owner socket to send more
2487  * packets, as if they were already TX completed by a typical driver.
2488  * But we also want to keep skb->sk set because some packet schedulers
2489  * rely on it (sch_fq for example).
2490  */
skb_orphan_partial(struct sk_buff * skb)2491 void skb_orphan_partial(struct sk_buff *skb)
2492 {
2493 	if (skb_is_tcp_pure_ack(skb))
2494 		return;
2495 
2496 	if (can_skb_orphan_partial(skb) && skb_set_owner_sk_safe(skb, skb->sk))
2497 		return;
2498 
2499 	skb_orphan(skb);
2500 }
2501 EXPORT_SYMBOL(skb_orphan_partial);
2502 
2503 /*
2504  * Read buffer destructor automatically called from kfree_skb.
2505  */
sock_rfree(struct sk_buff * skb)2506 void sock_rfree(struct sk_buff *skb)
2507 {
2508 	struct sock *sk = skb->sk;
2509 	unsigned int len = skb->truesize;
2510 
2511 	atomic_sub(len, &sk->sk_rmem_alloc);
2512 	sk_mem_uncharge(sk, len);
2513 }
2514 EXPORT_SYMBOL(sock_rfree);
2515 
2516 /*
2517  * Buffer destructor for skbs that are not used directly in read or write
2518  * path, e.g. for error handler skbs. Automatically called from kfree_skb.
2519  */
sock_efree(struct sk_buff * skb)2520 void sock_efree(struct sk_buff *skb)
2521 {
2522 	sock_put(skb->sk);
2523 }
2524 EXPORT_SYMBOL(sock_efree);
2525 
2526 /* Buffer destructor for prefetch/receive path where reference count may
2527  * not be held, e.g. for listen sockets.
2528  */
2529 #ifdef CONFIG_INET
sock_pfree(struct sk_buff * skb)2530 void sock_pfree(struct sk_buff *skb)
2531 {
2532 	if (sk_is_refcounted(skb->sk))
2533 		sock_gen_put(skb->sk);
2534 }
2535 EXPORT_SYMBOL(sock_pfree);
2536 #endif /* CONFIG_INET */
2537 
sock_i_uid(struct sock * sk)2538 kuid_t sock_i_uid(struct sock *sk)
2539 {
2540 	kuid_t uid;
2541 
2542 	read_lock_bh(&sk->sk_callback_lock);
2543 	uid = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_uid : GLOBAL_ROOT_UID;
2544 	read_unlock_bh(&sk->sk_callback_lock);
2545 	return uid;
2546 }
2547 EXPORT_SYMBOL(sock_i_uid);
2548 
sock_i_ino(struct sock * sk)2549 unsigned long sock_i_ino(struct sock *sk)
2550 {
2551 	unsigned long ino;
2552 
2553 	read_lock_bh(&sk->sk_callback_lock);
2554 	ino = sk->sk_socket ? SOCK_INODE(sk->sk_socket)->i_ino : 0;
2555 	read_unlock_bh(&sk->sk_callback_lock);
2556 	return ino;
2557 }
2558 EXPORT_SYMBOL(sock_i_ino);
2559 
2560 /*
2561  * Allocate a skb from the socket's send buffer.
2562  */
sock_wmalloc(struct sock * sk,unsigned long size,int force,gfp_t priority)2563 struct sk_buff *sock_wmalloc(struct sock *sk, unsigned long size, int force,
2564 			     gfp_t priority)
2565 {
2566 	if (force ||
2567 	    refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf)) {
2568 		struct sk_buff *skb = alloc_skb(size, priority);
2569 
2570 		if (skb) {
2571 			skb_set_owner_w(skb, sk);
2572 			return skb;
2573 		}
2574 	}
2575 	return NULL;
2576 }
2577 EXPORT_SYMBOL(sock_wmalloc);
2578 
sock_ofree(struct sk_buff * skb)2579 static void sock_ofree(struct sk_buff *skb)
2580 {
2581 	struct sock *sk = skb->sk;
2582 
2583 	atomic_sub(skb->truesize, &sk->sk_omem_alloc);
2584 }
2585 
sock_omalloc(struct sock * sk,unsigned long size,gfp_t priority)2586 struct sk_buff *sock_omalloc(struct sock *sk, unsigned long size,
2587 			     gfp_t priority)
2588 {
2589 	struct sk_buff *skb;
2590 
2591 	/* small safe race: SKB_TRUESIZE may differ from final skb->truesize */
2592 	if (atomic_read(&sk->sk_omem_alloc) + SKB_TRUESIZE(size) >
2593 	    READ_ONCE(sysctl_optmem_max))
2594 		return NULL;
2595 
2596 	skb = alloc_skb(size, priority);
2597 	if (!skb)
2598 		return NULL;
2599 
2600 	atomic_add(skb->truesize, &sk->sk_omem_alloc);
2601 	skb->sk = sk;
2602 	skb->destructor = sock_ofree;
2603 	return skb;
2604 }
2605 
2606 /*
2607  * Allocate a memory block from the socket's option memory buffer.
2608  */
sock_kmalloc(struct sock * sk,int size,gfp_t priority)2609 void *sock_kmalloc(struct sock *sk, int size, gfp_t priority)
2610 {
2611 	int optmem_max = READ_ONCE(sysctl_optmem_max);
2612 
2613 	if ((unsigned int)size <= optmem_max &&
2614 	    atomic_read(&sk->sk_omem_alloc) + size < optmem_max) {
2615 		void *mem;
2616 		/* First do the add, to avoid the race if kmalloc
2617 		 * might sleep.
2618 		 */
2619 		atomic_add(size, &sk->sk_omem_alloc);
2620 		mem = kmalloc(size, priority);
2621 		if (mem)
2622 			return mem;
2623 		atomic_sub(size, &sk->sk_omem_alloc);
2624 	}
2625 	return NULL;
2626 }
2627 EXPORT_SYMBOL(sock_kmalloc);
2628 
2629 /* Free an option memory block. Note, we actually want the inline
2630  * here as this allows gcc to detect the nullify and fold away the
2631  * condition entirely.
2632  */
__sock_kfree_s(struct sock * sk,void * mem,int size,const bool nullify)2633 static inline void __sock_kfree_s(struct sock *sk, void *mem, int size,
2634 				  const bool nullify)
2635 {
2636 	if (WARN_ON_ONCE(!mem))
2637 		return;
2638 	if (nullify)
2639 		kfree_sensitive(mem);
2640 	else
2641 		kfree(mem);
2642 	atomic_sub(size, &sk->sk_omem_alloc);
2643 }
2644 
sock_kfree_s(struct sock * sk,void * mem,int size)2645 void sock_kfree_s(struct sock *sk, void *mem, int size)
2646 {
2647 	__sock_kfree_s(sk, mem, size, false);
2648 }
2649 EXPORT_SYMBOL(sock_kfree_s);
2650 
sock_kzfree_s(struct sock * sk,void * mem,int size)2651 void sock_kzfree_s(struct sock *sk, void *mem, int size)
2652 {
2653 	__sock_kfree_s(sk, mem, size, true);
2654 }
2655 EXPORT_SYMBOL(sock_kzfree_s);
2656 
2657 /* It is almost wait_for_tcp_memory minus release_sock/lock_sock.
2658    I think, these locks should be removed for datagram sockets.
2659  */
sock_wait_for_wmem(struct sock * sk,long timeo)2660 static long sock_wait_for_wmem(struct sock *sk, long timeo)
2661 {
2662 	DEFINE_WAIT(wait);
2663 
2664 	sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2665 	for (;;) {
2666 		if (!timeo)
2667 			break;
2668 		if (signal_pending(current))
2669 			break;
2670 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2671 		prepare_to_wait(sk_sleep(sk), &wait, TASK_INTERRUPTIBLE);
2672 		if (refcount_read(&sk->sk_wmem_alloc) < READ_ONCE(sk->sk_sndbuf))
2673 			break;
2674 		if (sk->sk_shutdown & SEND_SHUTDOWN)
2675 			break;
2676 		if (sk->sk_err)
2677 			break;
2678 		timeo = schedule_timeout(timeo);
2679 	}
2680 	finish_wait(sk_sleep(sk), &wait);
2681 	return timeo;
2682 }
2683 
2684 
2685 /*
2686  *	Generic send/receive buffer handlers
2687  */
2688 
sock_alloc_send_pskb(struct sock * sk,unsigned long header_len,unsigned long data_len,int noblock,int * errcode,int max_page_order)2689 struct sk_buff *sock_alloc_send_pskb(struct sock *sk, unsigned long header_len,
2690 				     unsigned long data_len, int noblock,
2691 				     int *errcode, int max_page_order)
2692 {
2693 	struct sk_buff *skb;
2694 	long timeo;
2695 	int err;
2696 
2697 	timeo = sock_sndtimeo(sk, noblock);
2698 	for (;;) {
2699 		err = sock_error(sk);
2700 		if (err != 0)
2701 			goto failure;
2702 
2703 		err = -EPIPE;
2704 		if (sk->sk_shutdown & SEND_SHUTDOWN)
2705 			goto failure;
2706 
2707 		if (sk_wmem_alloc_get(sk) < READ_ONCE(sk->sk_sndbuf))
2708 			break;
2709 
2710 		sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
2711 		set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
2712 		err = -EAGAIN;
2713 		if (!timeo)
2714 			goto failure;
2715 		if (signal_pending(current))
2716 			goto interrupted;
2717 		timeo = sock_wait_for_wmem(sk, timeo);
2718 	}
2719 	skb = alloc_skb_with_frags(header_len, data_len, max_page_order,
2720 				   errcode, sk->sk_allocation);
2721 	if (skb)
2722 		skb_set_owner_w(skb, sk);
2723 	return skb;
2724 
2725 interrupted:
2726 	err = sock_intr_errno(timeo);
2727 failure:
2728 	*errcode = err;
2729 	return NULL;
2730 }
2731 EXPORT_SYMBOL(sock_alloc_send_pskb);
2732 
__sock_cmsg_send(struct sock * sk,struct msghdr * msg,struct cmsghdr * cmsg,struct sockcm_cookie * sockc)2733 int __sock_cmsg_send(struct sock *sk, struct msghdr *msg, struct cmsghdr *cmsg,
2734 		     struct sockcm_cookie *sockc)
2735 {
2736 	u32 tsflags;
2737 
2738 	switch (cmsg->cmsg_type) {
2739 	case SO_MARK:
2740 		if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_RAW) &&
2741 		    !ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN))
2742 			return -EPERM;
2743 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2744 			return -EINVAL;
2745 		sockc->mark = *(u32 *)CMSG_DATA(cmsg);
2746 		break;
2747 	case SO_TIMESTAMPING_OLD:
2748 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u32)))
2749 			return -EINVAL;
2750 
2751 		tsflags = *(u32 *)CMSG_DATA(cmsg);
2752 		if (tsflags & ~SOF_TIMESTAMPING_TX_RECORD_MASK)
2753 			return -EINVAL;
2754 
2755 		sockc->tsflags &= ~SOF_TIMESTAMPING_TX_RECORD_MASK;
2756 		sockc->tsflags |= tsflags;
2757 		break;
2758 	case SCM_TXTIME:
2759 		if (!sock_flag(sk, SOCK_TXTIME))
2760 			return -EINVAL;
2761 		if (cmsg->cmsg_len != CMSG_LEN(sizeof(u64)))
2762 			return -EINVAL;
2763 		sockc->transmit_time = get_unaligned((u64 *)CMSG_DATA(cmsg));
2764 		break;
2765 	/* SCM_RIGHTS and SCM_CREDENTIALS are semantically in SOL_UNIX. */
2766 	case SCM_RIGHTS:
2767 	case SCM_CREDENTIALS:
2768 		break;
2769 	default:
2770 		return -EINVAL;
2771 	}
2772 	return 0;
2773 }
2774 EXPORT_SYMBOL(__sock_cmsg_send);
2775 
sock_cmsg_send(struct sock * sk,struct msghdr * msg,struct sockcm_cookie * sockc)2776 int sock_cmsg_send(struct sock *sk, struct msghdr *msg,
2777 		   struct sockcm_cookie *sockc)
2778 {
2779 	struct cmsghdr *cmsg;
2780 	int ret;
2781 
2782 	for_each_cmsghdr(cmsg, msg) {
2783 		if (!CMSG_OK(msg, cmsg))
2784 			return -EINVAL;
2785 		if (cmsg->cmsg_level != SOL_SOCKET)
2786 			continue;
2787 		ret = __sock_cmsg_send(sk, msg, cmsg, sockc);
2788 		if (ret)
2789 			return ret;
2790 	}
2791 	return 0;
2792 }
2793 EXPORT_SYMBOL(sock_cmsg_send);
2794 
sk_enter_memory_pressure(struct sock * sk)2795 static void sk_enter_memory_pressure(struct sock *sk)
2796 {
2797 	if (!sk->sk_prot->enter_memory_pressure)
2798 		return;
2799 
2800 	sk->sk_prot->enter_memory_pressure(sk);
2801 }
2802 
sk_leave_memory_pressure(struct sock * sk)2803 static void sk_leave_memory_pressure(struct sock *sk)
2804 {
2805 	if (sk->sk_prot->leave_memory_pressure) {
2806 		sk->sk_prot->leave_memory_pressure(sk);
2807 	} else {
2808 		unsigned long *memory_pressure = sk->sk_prot->memory_pressure;
2809 
2810 		if (memory_pressure && READ_ONCE(*memory_pressure))
2811 			WRITE_ONCE(*memory_pressure, 0);
2812 	}
2813 }
2814 
2815 DEFINE_STATIC_KEY_FALSE(net_high_order_alloc_disable_key);
2816 
2817 /**
2818  * skb_page_frag_refill - check that a page_frag contains enough room
2819  * @sz: minimum size of the fragment we want to get
2820  * @pfrag: pointer to page_frag
2821  * @gfp: priority for memory allocation
2822  *
2823  * Note: While this allocator tries to use high order pages, there is
2824  * no guarantee that allocations succeed. Therefore, @sz MUST be
2825  * less or equal than PAGE_SIZE.
2826  */
skb_page_frag_refill(unsigned int sz,struct page_frag * pfrag,gfp_t gfp)2827 bool skb_page_frag_refill(unsigned int sz, struct page_frag *pfrag, gfp_t gfp)
2828 {
2829 	if (pfrag->page) {
2830 		if (page_ref_count(pfrag->page) == 1) {
2831 			pfrag->offset = 0;
2832 			return true;
2833 		}
2834 		if (pfrag->offset + sz <= pfrag->size)
2835 			return true;
2836 		put_page(pfrag->page);
2837 	}
2838 
2839 	pfrag->offset = 0;
2840 	if (SKB_FRAG_PAGE_ORDER &&
2841 	    !static_branch_unlikely(&net_high_order_alloc_disable_key)) {
2842 		/* Avoid direct reclaim but allow kswapd to wake */
2843 		pfrag->page = alloc_pages((gfp & ~__GFP_DIRECT_RECLAIM) |
2844 					  __GFP_COMP | __GFP_NOWARN |
2845 					  __GFP_NORETRY,
2846 					  SKB_FRAG_PAGE_ORDER);
2847 		if (likely(pfrag->page)) {
2848 			pfrag->size = PAGE_SIZE << SKB_FRAG_PAGE_ORDER;
2849 			return true;
2850 		}
2851 	}
2852 	pfrag->page = alloc_page(gfp);
2853 	if (likely(pfrag->page)) {
2854 		pfrag->size = PAGE_SIZE;
2855 		return true;
2856 	}
2857 	return false;
2858 }
2859 EXPORT_SYMBOL(skb_page_frag_refill);
2860 
sk_page_frag_refill(struct sock * sk,struct page_frag * pfrag)2861 bool sk_page_frag_refill(struct sock *sk, struct page_frag *pfrag)
2862 {
2863 	if (likely(skb_page_frag_refill(32U, pfrag, sk->sk_allocation)))
2864 		return true;
2865 
2866 	sk_enter_memory_pressure(sk);
2867 	sk_stream_moderate_sndbuf(sk);
2868 	return false;
2869 }
2870 EXPORT_SYMBOL(sk_page_frag_refill);
2871 
__lock_sock(struct sock * sk)2872 void __lock_sock(struct sock *sk)
2873 	__releases(&sk->sk_lock.slock)
2874 	__acquires(&sk->sk_lock.slock)
2875 {
2876 	DEFINE_WAIT(wait);
2877 
2878 	for (;;) {
2879 		prepare_to_wait_exclusive(&sk->sk_lock.wq, &wait,
2880 					TASK_UNINTERRUPTIBLE);
2881 		spin_unlock_bh(&sk->sk_lock.slock);
2882 		schedule();
2883 		spin_lock_bh(&sk->sk_lock.slock);
2884 		if (!sock_owned_by_user(sk))
2885 			break;
2886 	}
2887 	finish_wait(&sk->sk_lock.wq, &wait);
2888 }
2889 
__release_sock(struct sock * sk)2890 void __release_sock(struct sock *sk)
2891 	__releases(&sk->sk_lock.slock)
2892 	__acquires(&sk->sk_lock.slock)
2893 {
2894 	struct sk_buff *skb, *next;
2895 
2896 	while ((skb = sk->sk_backlog.head) != NULL) {
2897 		sk->sk_backlog.head = sk->sk_backlog.tail = NULL;
2898 
2899 		spin_unlock_bh(&sk->sk_lock.slock);
2900 
2901 		do {
2902 			next = skb->next;
2903 			prefetch(next);
2904 			DEBUG_NET_WARN_ON_ONCE(skb_dst_is_noref(skb));
2905 			skb_mark_not_on_list(skb);
2906 			sk_backlog_rcv(sk, skb);
2907 
2908 			cond_resched();
2909 
2910 			skb = next;
2911 		} while (skb != NULL);
2912 
2913 		spin_lock_bh(&sk->sk_lock.slock);
2914 	}
2915 
2916 	/*
2917 	 * Doing the zeroing here guarantee we can not loop forever
2918 	 * while a wild producer attempts to flood us.
2919 	 */
2920 	sk->sk_backlog.len = 0;
2921 }
2922 
__sk_flush_backlog(struct sock * sk)2923 void __sk_flush_backlog(struct sock *sk)
2924 {
2925 	spin_lock_bh(&sk->sk_lock.slock);
2926 	__release_sock(sk);
2927 	spin_unlock_bh(&sk->sk_lock.slock);
2928 }
2929 EXPORT_SYMBOL_GPL(__sk_flush_backlog);
2930 
2931 /**
2932  * sk_wait_data - wait for data to arrive at sk_receive_queue
2933  * @sk:    sock to wait on
2934  * @timeo: for how long
2935  * @skb:   last skb seen on sk_receive_queue
2936  *
2937  * Now socket state including sk->sk_err is changed only under lock,
2938  * hence we may omit checks after joining wait queue.
2939  * We check receive queue before schedule() only as optimization;
2940  * it is very likely that release_sock() added new data.
2941  */
sk_wait_data(struct sock * sk,long * timeo,const struct sk_buff * skb)2942 int sk_wait_data(struct sock *sk, long *timeo, const struct sk_buff *skb)
2943 {
2944 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
2945 	int rc;
2946 
2947 	add_wait_queue(sk_sleep(sk), &wait);
2948 	sk_set_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2949 	rc = sk_wait_event(sk, timeo, skb_peek_tail(&sk->sk_receive_queue) != skb, &wait);
2950 	sk_clear_bit(SOCKWQ_ASYNC_WAITDATA, sk);
2951 	remove_wait_queue(sk_sleep(sk), &wait);
2952 	return rc;
2953 }
2954 EXPORT_SYMBOL(sk_wait_data);
2955 
2956 /**
2957  *	__sk_mem_raise_allocated - increase memory_allocated
2958  *	@sk: socket
2959  *	@size: memory size to allocate
2960  *	@amt: pages to allocate
2961  *	@kind: allocation type
2962  *
2963  *	Similar to __sk_mem_schedule(), but does not update sk_forward_alloc
2964  */
__sk_mem_raise_allocated(struct sock * sk,int size,int amt,int kind)2965 int __sk_mem_raise_allocated(struct sock *sk, int size, int amt, int kind)
2966 {
2967 	bool memcg_charge = mem_cgroup_sockets_enabled && sk->sk_memcg;
2968 	struct proto *prot = sk->sk_prot;
2969 	bool charged = true;
2970 	long allocated;
2971 
2972 	sk_memory_allocated_add(sk, amt);
2973 	allocated = sk_memory_allocated(sk);
2974 	if (memcg_charge &&
2975 	    !(charged = mem_cgroup_charge_skmem(sk->sk_memcg, amt,
2976 						gfp_memcg_charge())))
2977 		goto suppress_allocation;
2978 
2979 	/* Under limit. */
2980 	if (allocated <= sk_prot_mem_limits(sk, 0)) {
2981 		sk_leave_memory_pressure(sk);
2982 		return 1;
2983 	}
2984 
2985 	/* Under pressure. */
2986 	if (allocated > sk_prot_mem_limits(sk, 1))
2987 		sk_enter_memory_pressure(sk);
2988 
2989 	/* Over hard limit. */
2990 	if (allocated > sk_prot_mem_limits(sk, 2))
2991 		goto suppress_allocation;
2992 
2993 	/* guarantee minimum buffer size under pressure */
2994 	if (kind == SK_MEM_RECV) {
2995 		if (atomic_read(&sk->sk_rmem_alloc) < sk_get_rmem0(sk, prot))
2996 			return 1;
2997 
2998 	} else { /* SK_MEM_SEND */
2999 		int wmem0 = sk_get_wmem0(sk, prot);
3000 
3001 		if (sk->sk_type == SOCK_STREAM) {
3002 			if (sk->sk_wmem_queued < wmem0)
3003 				return 1;
3004 		} else if (refcount_read(&sk->sk_wmem_alloc) < wmem0) {
3005 				return 1;
3006 		}
3007 	}
3008 
3009 	if (sk_has_memory_pressure(sk)) {
3010 		u64 alloc;
3011 
3012 		if (!sk_under_memory_pressure(sk))
3013 			return 1;
3014 		alloc = sk_sockets_allocated_read_positive(sk);
3015 		if (sk_prot_mem_limits(sk, 2) > alloc *
3016 		    sk_mem_pages(sk->sk_wmem_queued +
3017 				 atomic_read(&sk->sk_rmem_alloc) +
3018 				 sk->sk_forward_alloc))
3019 			return 1;
3020 	}
3021 
3022 suppress_allocation:
3023 
3024 	if (kind == SK_MEM_SEND && sk->sk_type == SOCK_STREAM) {
3025 		sk_stream_moderate_sndbuf(sk);
3026 
3027 		/* Fail only if socket is _under_ its sndbuf.
3028 		 * In this case we cannot block, so that we have to fail.
3029 		 */
3030 		if (sk->sk_wmem_queued + size >= sk->sk_sndbuf) {
3031 			/* Force charge with __GFP_NOFAIL */
3032 			if (memcg_charge && !charged) {
3033 				mem_cgroup_charge_skmem(sk->sk_memcg, amt,
3034 					gfp_memcg_charge() | __GFP_NOFAIL);
3035 			}
3036 			return 1;
3037 		}
3038 	}
3039 
3040 	if (kind == SK_MEM_SEND || (kind == SK_MEM_RECV && charged))
3041 		trace_sock_exceed_buf_limit(sk, prot, allocated, kind);
3042 
3043 	sk_memory_allocated_sub(sk, amt);
3044 
3045 	if (memcg_charge && charged)
3046 		mem_cgroup_uncharge_skmem(sk->sk_memcg, amt);
3047 
3048 	return 0;
3049 }
3050 
3051 /**
3052  *	__sk_mem_schedule - increase sk_forward_alloc and memory_allocated
3053  *	@sk: socket
3054  *	@size: memory size to allocate
3055  *	@kind: allocation type
3056  *
3057  *	If kind is SK_MEM_SEND, it means wmem allocation. Otherwise it means
3058  *	rmem allocation. This function assumes that protocols which have
3059  *	memory_pressure use sk_wmem_queued as write buffer accounting.
3060  */
__sk_mem_schedule(struct sock * sk,int size,int kind)3061 int __sk_mem_schedule(struct sock *sk, int size, int kind)
3062 {
3063 	int ret, amt = sk_mem_pages(size);
3064 
3065 	sk->sk_forward_alloc += amt << PAGE_SHIFT;
3066 	ret = __sk_mem_raise_allocated(sk, size, amt, kind);
3067 	if (!ret)
3068 		sk->sk_forward_alloc -= amt << PAGE_SHIFT;
3069 	return ret;
3070 }
3071 EXPORT_SYMBOL(__sk_mem_schedule);
3072 
3073 /**
3074  *	__sk_mem_reduce_allocated - reclaim memory_allocated
3075  *	@sk: socket
3076  *	@amount: number of quanta
3077  *
3078  *	Similar to __sk_mem_reclaim(), but does not update sk_forward_alloc
3079  */
__sk_mem_reduce_allocated(struct sock * sk,int amount)3080 void __sk_mem_reduce_allocated(struct sock *sk, int amount)
3081 {
3082 	sk_memory_allocated_sub(sk, amount);
3083 
3084 	if (mem_cgroup_sockets_enabled && sk->sk_memcg)
3085 		mem_cgroup_uncharge_skmem(sk->sk_memcg, amount);
3086 
3087 	if (sk_under_memory_pressure(sk) &&
3088 	    (sk_memory_allocated(sk) < sk_prot_mem_limits(sk, 0)))
3089 		sk_leave_memory_pressure(sk);
3090 }
3091 
3092 /**
3093  *	__sk_mem_reclaim - reclaim sk_forward_alloc and memory_allocated
3094  *	@sk: socket
3095  *	@amount: number of bytes (rounded down to a PAGE_SIZE multiple)
3096  */
__sk_mem_reclaim(struct sock * sk,int amount)3097 void __sk_mem_reclaim(struct sock *sk, int amount)
3098 {
3099 	amount >>= PAGE_SHIFT;
3100 	sk->sk_forward_alloc -= amount << PAGE_SHIFT;
3101 	__sk_mem_reduce_allocated(sk, amount);
3102 }
3103 EXPORT_SYMBOL(__sk_mem_reclaim);
3104 
sk_set_peek_off(struct sock * sk,int val)3105 int sk_set_peek_off(struct sock *sk, int val)
3106 {
3107 	sk->sk_peek_off = val;
3108 	return 0;
3109 }
3110 EXPORT_SYMBOL_GPL(sk_set_peek_off);
3111 
3112 /*
3113  * Set of default routines for initialising struct proto_ops when
3114  * the protocol does not support a particular function. In certain
3115  * cases where it makes no sense for a protocol to have a "do nothing"
3116  * function, some default processing is provided.
3117  */
3118 
sock_no_bind(struct socket * sock,struct sockaddr * saddr,int len)3119 int sock_no_bind(struct socket *sock, struct sockaddr *saddr, int len)
3120 {
3121 	return -EOPNOTSUPP;
3122 }
3123 EXPORT_SYMBOL(sock_no_bind);
3124 
sock_no_connect(struct socket * sock,struct sockaddr * saddr,int len,int flags)3125 int sock_no_connect(struct socket *sock, struct sockaddr *saddr,
3126 		    int len, int flags)
3127 {
3128 	return -EOPNOTSUPP;
3129 }
3130 EXPORT_SYMBOL(sock_no_connect);
3131 
sock_no_socketpair(struct socket * sock1,struct socket * sock2)3132 int sock_no_socketpair(struct socket *sock1, struct socket *sock2)
3133 {
3134 	return -EOPNOTSUPP;
3135 }
3136 EXPORT_SYMBOL(sock_no_socketpair);
3137 
sock_no_accept(struct socket * sock,struct socket * newsock,int flags,bool kern)3138 int sock_no_accept(struct socket *sock, struct socket *newsock, int flags,
3139 		   bool kern)
3140 {
3141 	return -EOPNOTSUPP;
3142 }
3143 EXPORT_SYMBOL(sock_no_accept);
3144 
sock_no_getname(struct socket * sock,struct sockaddr * saddr,int peer)3145 int sock_no_getname(struct socket *sock, struct sockaddr *saddr,
3146 		    int peer)
3147 {
3148 	return -EOPNOTSUPP;
3149 }
3150 EXPORT_SYMBOL(sock_no_getname);
3151 
sock_no_ioctl(struct socket * sock,unsigned int cmd,unsigned long arg)3152 int sock_no_ioctl(struct socket *sock, unsigned int cmd, unsigned long arg)
3153 {
3154 	return -EOPNOTSUPP;
3155 }
3156 EXPORT_SYMBOL(sock_no_ioctl);
3157 
sock_no_listen(struct socket * sock,int backlog)3158 int sock_no_listen(struct socket *sock, int backlog)
3159 {
3160 	return -EOPNOTSUPP;
3161 }
3162 EXPORT_SYMBOL(sock_no_listen);
3163 
sock_no_shutdown(struct socket * sock,int how)3164 int sock_no_shutdown(struct socket *sock, int how)
3165 {
3166 	return -EOPNOTSUPP;
3167 }
3168 EXPORT_SYMBOL(sock_no_shutdown);
3169 
sock_no_sendmsg(struct socket * sock,struct msghdr * m,size_t len)3170 int sock_no_sendmsg(struct socket *sock, struct msghdr *m, size_t len)
3171 {
3172 	return -EOPNOTSUPP;
3173 }
3174 EXPORT_SYMBOL(sock_no_sendmsg);
3175 
sock_no_sendmsg_locked(struct sock * sk,struct msghdr * m,size_t len)3176 int sock_no_sendmsg_locked(struct sock *sk, struct msghdr *m, size_t len)
3177 {
3178 	return -EOPNOTSUPP;
3179 }
3180 EXPORT_SYMBOL(sock_no_sendmsg_locked);
3181 
sock_no_recvmsg(struct socket * sock,struct msghdr * m,size_t len,int flags)3182 int sock_no_recvmsg(struct socket *sock, struct msghdr *m, size_t len,
3183 		    int flags)
3184 {
3185 	return -EOPNOTSUPP;
3186 }
3187 EXPORT_SYMBOL(sock_no_recvmsg);
3188 
sock_no_mmap(struct file * file,struct socket * sock,struct vm_area_struct * vma)3189 int sock_no_mmap(struct file *file, struct socket *sock, struct vm_area_struct *vma)
3190 {
3191 	/* Mirror missing mmap method error code */
3192 	return -ENODEV;
3193 }
3194 EXPORT_SYMBOL(sock_no_mmap);
3195 
3196 /*
3197  * When a file is received (via SCM_RIGHTS, etc), we must bump the
3198  * various sock-based usage counts.
3199  */
__receive_sock(struct file * file)3200 void __receive_sock(struct file *file)
3201 {
3202 	struct socket *sock;
3203 
3204 	sock = sock_from_file(file);
3205 	if (sock) {
3206 		sock_update_netprioidx(&sock->sk->sk_cgrp_data);
3207 		sock_update_classid(&sock->sk->sk_cgrp_data);
3208 	}
3209 }
3210 
sock_no_sendpage(struct socket * sock,struct page * page,int offset,size_t size,int flags)3211 ssize_t sock_no_sendpage(struct socket *sock, struct page *page, int offset, size_t size, int flags)
3212 {
3213 	ssize_t res;
3214 	struct msghdr msg = {.msg_flags = flags};
3215 	struct kvec iov;
3216 	char *kaddr = kmap(page);
3217 	iov.iov_base = kaddr + offset;
3218 	iov.iov_len = size;
3219 	res = kernel_sendmsg(sock, &msg, &iov, 1, size);
3220 	kunmap(page);
3221 	return res;
3222 }
3223 EXPORT_SYMBOL(sock_no_sendpage);
3224 
sock_no_sendpage_locked(struct sock * sk,struct page * page,int offset,size_t size,int flags)3225 ssize_t sock_no_sendpage_locked(struct sock *sk, struct page *page,
3226 				int offset, size_t size, int flags)
3227 {
3228 	ssize_t res;
3229 	struct msghdr msg = {.msg_flags = flags};
3230 	struct kvec iov;
3231 	char *kaddr = kmap(page);
3232 
3233 	iov.iov_base = kaddr + offset;
3234 	iov.iov_len = size;
3235 	res = kernel_sendmsg_locked(sk, &msg, &iov, 1, size);
3236 	kunmap(page);
3237 	return res;
3238 }
3239 EXPORT_SYMBOL(sock_no_sendpage_locked);
3240 
3241 /*
3242  *	Default Socket Callbacks
3243  */
3244 
sock_def_wakeup(struct sock * sk)3245 static void sock_def_wakeup(struct sock *sk)
3246 {
3247 	struct socket_wq *wq;
3248 
3249 	rcu_read_lock();
3250 	wq = rcu_dereference(sk->sk_wq);
3251 	if (skwq_has_sleeper(wq))
3252 		wake_up_interruptible_all(&wq->wait);
3253 	rcu_read_unlock();
3254 }
3255 
sock_def_error_report(struct sock * sk)3256 static void sock_def_error_report(struct sock *sk)
3257 {
3258 	struct socket_wq *wq;
3259 
3260 	rcu_read_lock();
3261 	wq = rcu_dereference(sk->sk_wq);
3262 	if (skwq_has_sleeper(wq))
3263 		wake_up_interruptible_poll(&wq->wait, EPOLLERR);
3264 	sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
3265 	rcu_read_unlock();
3266 }
3267 
sock_def_readable(struct sock * sk)3268 void sock_def_readable(struct sock *sk)
3269 {
3270 	struct socket_wq *wq;
3271 
3272 	rcu_read_lock();
3273 	wq = rcu_dereference(sk->sk_wq);
3274 	if (skwq_has_sleeper(wq))
3275 		wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
3276 						EPOLLRDNORM | EPOLLRDBAND);
3277 	sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
3278 	rcu_read_unlock();
3279 }
3280 
sock_def_write_space(struct sock * sk)3281 static void sock_def_write_space(struct sock *sk)
3282 {
3283 	struct socket_wq *wq;
3284 
3285 	rcu_read_lock();
3286 
3287 	/* Do not wake up a writer until he can make "significant"
3288 	 * progress.  --DaveM
3289 	 */
3290 	if (sock_writeable(sk)) {
3291 		wq = rcu_dereference(sk->sk_wq);
3292 		if (skwq_has_sleeper(wq))
3293 			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3294 						EPOLLWRNORM | EPOLLWRBAND);
3295 
3296 		/* Should agree with poll, otherwise some programs break */
3297 		sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
3298 	}
3299 
3300 	rcu_read_unlock();
3301 }
3302 
3303 /* An optimised version of sock_def_write_space(), should only be called
3304  * for SOCK_RCU_FREE sockets under RCU read section and after putting
3305  * ->sk_wmem_alloc.
3306  */
sock_def_write_space_wfree(struct sock * sk)3307 static void sock_def_write_space_wfree(struct sock *sk)
3308 {
3309 	/* Do not wake up a writer until he can make "significant"
3310 	 * progress.  --DaveM
3311 	 */
3312 	if (sock_writeable(sk)) {
3313 		struct socket_wq *wq = rcu_dereference(sk->sk_wq);
3314 
3315 		/* rely on refcount_sub from sock_wfree() */
3316 		smp_mb__after_atomic();
3317 		if (wq && waitqueue_active(&wq->wait))
3318 			wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
3319 						EPOLLWRNORM | EPOLLWRBAND);
3320 
3321 		/* Should agree with poll, otherwise some programs break */
3322 		sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
3323 	}
3324 }
3325 
sock_def_destruct(struct sock * sk)3326 static void sock_def_destruct(struct sock *sk)
3327 {
3328 }
3329 
sk_send_sigurg(struct sock * sk)3330 void sk_send_sigurg(struct sock *sk)
3331 {
3332 	if (sk->sk_socket && sk->sk_socket->file)
3333 		if (send_sigurg(&sk->sk_socket->file->f_owner))
3334 			sk_wake_async(sk, SOCK_WAKE_URG, POLL_PRI);
3335 }
3336 EXPORT_SYMBOL(sk_send_sigurg);
3337 
sk_reset_timer(struct sock * sk,struct timer_list * timer,unsigned long expires)3338 void sk_reset_timer(struct sock *sk, struct timer_list* timer,
3339 		    unsigned long expires)
3340 {
3341 	if (!mod_timer(timer, expires))
3342 		sock_hold(sk);
3343 }
3344 EXPORT_SYMBOL(sk_reset_timer);
3345 
sk_stop_timer(struct sock * sk,struct timer_list * timer)3346 void sk_stop_timer(struct sock *sk, struct timer_list* timer)
3347 {
3348 	if (del_timer(timer))
3349 		__sock_put(sk);
3350 }
3351 EXPORT_SYMBOL(sk_stop_timer);
3352 
sk_stop_timer_sync(struct sock * sk,struct timer_list * timer)3353 void sk_stop_timer_sync(struct sock *sk, struct timer_list *timer)
3354 {
3355 	if (del_timer_sync(timer))
3356 		__sock_put(sk);
3357 }
3358 EXPORT_SYMBOL(sk_stop_timer_sync);
3359 
sock_init_data(struct socket * sock,struct sock * sk)3360 void sock_init_data(struct socket *sock, struct sock *sk)
3361 {
3362 	sk_init_common(sk);
3363 	sk->sk_send_head	=	NULL;
3364 
3365 	timer_setup(&sk->sk_timer, NULL, 0);
3366 
3367 	sk->sk_allocation	=	GFP_KERNEL;
3368 	sk->sk_rcvbuf		=	READ_ONCE(sysctl_rmem_default);
3369 	sk->sk_sndbuf		=	READ_ONCE(sysctl_wmem_default);
3370 	sk->sk_state		=	TCP_CLOSE;
3371 	sk_set_socket(sk, sock);
3372 
3373 	sock_set_flag(sk, SOCK_ZAPPED);
3374 
3375 	if (sock) {
3376 		sk->sk_type	=	sock->type;
3377 		RCU_INIT_POINTER(sk->sk_wq, &sock->wq);
3378 		sock->sk	=	sk;
3379 		sk->sk_uid	=	SOCK_INODE(sock)->i_uid;
3380 	} else {
3381 		RCU_INIT_POINTER(sk->sk_wq, NULL);
3382 		sk->sk_uid	=	make_kuid(sock_net(sk)->user_ns, 0);
3383 	}
3384 
3385 	rwlock_init(&sk->sk_callback_lock);
3386 	if (sk->sk_kern_sock)
3387 		lockdep_set_class_and_name(
3388 			&sk->sk_callback_lock,
3389 			af_kern_callback_keys + sk->sk_family,
3390 			af_family_kern_clock_key_strings[sk->sk_family]);
3391 	else
3392 		lockdep_set_class_and_name(
3393 			&sk->sk_callback_lock,
3394 			af_callback_keys + sk->sk_family,
3395 			af_family_clock_key_strings[sk->sk_family]);
3396 
3397 	sk->sk_state_change	=	sock_def_wakeup;
3398 	sk->sk_data_ready	=	sock_def_readable;
3399 	sk->sk_write_space	=	sock_def_write_space;
3400 	sk->sk_error_report	=	sock_def_error_report;
3401 	sk->sk_destruct		=	sock_def_destruct;
3402 
3403 	sk->sk_frag.page	=	NULL;
3404 	sk->sk_frag.offset	=	0;
3405 	sk->sk_peek_off		=	-1;
3406 
3407 	sk->sk_peer_pid 	=	NULL;
3408 	sk->sk_peer_cred	=	NULL;
3409 	spin_lock_init(&sk->sk_peer_lock);
3410 
3411 	sk->sk_write_pending	=	0;
3412 	sk->sk_rcvlowat		=	1;
3413 	sk->sk_rcvtimeo		=	MAX_SCHEDULE_TIMEOUT;
3414 	sk->sk_sndtimeo		=	MAX_SCHEDULE_TIMEOUT;
3415 
3416 	sk->sk_stamp = SK_DEFAULT_STAMP;
3417 #if BITS_PER_LONG==32
3418 	seqlock_init(&sk->sk_stamp_seq);
3419 #endif
3420 	atomic_set(&sk->sk_zckey, 0);
3421 
3422 #ifdef CONFIG_NET_RX_BUSY_POLL
3423 	sk->sk_napi_id		=	0;
3424 	sk->sk_ll_usec		=	READ_ONCE(sysctl_net_busy_read);
3425 #endif
3426 
3427 	sk->sk_max_pacing_rate = ~0UL;
3428 	sk->sk_pacing_rate = ~0UL;
3429 	WRITE_ONCE(sk->sk_pacing_shift, 10);
3430 	sk->sk_incoming_cpu = -1;
3431 	sk->sk_txrehash = SOCK_TXREHASH_DEFAULT;
3432 
3433 	sk_rx_queue_clear(sk);
3434 	/*
3435 	 * Before updating sk_refcnt, we must commit prior changes to memory
3436 	 * (Documentation/RCU/rculist_nulls.rst for details)
3437 	 */
3438 	smp_wmb();
3439 	refcount_set(&sk->sk_refcnt, 1);
3440 	atomic_set(&sk->sk_drops, 0);
3441 }
3442 EXPORT_SYMBOL(sock_init_data);
3443 
lock_sock_nested(struct sock * sk,int subclass)3444 void lock_sock_nested(struct sock *sk, int subclass)
3445 {
3446 	/* The sk_lock has mutex_lock() semantics here. */
3447 	mutex_acquire(&sk->sk_lock.dep_map, subclass, 0, _RET_IP_);
3448 
3449 	might_sleep();
3450 	spin_lock_bh(&sk->sk_lock.slock);
3451 	if (sock_owned_by_user_nocheck(sk))
3452 		__lock_sock(sk);
3453 	sk->sk_lock.owned = 1;
3454 	spin_unlock_bh(&sk->sk_lock.slock);
3455 }
3456 EXPORT_SYMBOL(lock_sock_nested);
3457 
release_sock(struct sock * sk)3458 void release_sock(struct sock *sk)
3459 {
3460 	spin_lock_bh(&sk->sk_lock.slock);
3461 	if (sk->sk_backlog.tail)
3462 		__release_sock(sk);
3463 
3464 	/* Warning : release_cb() might need to release sk ownership,
3465 	 * ie call sock_release_ownership(sk) before us.
3466 	 */
3467 	if (sk->sk_prot->release_cb)
3468 		sk->sk_prot->release_cb(sk);
3469 
3470 	sock_release_ownership(sk);
3471 	if (waitqueue_active(&sk->sk_lock.wq))
3472 		wake_up(&sk->sk_lock.wq);
3473 	spin_unlock_bh(&sk->sk_lock.slock);
3474 }
3475 EXPORT_SYMBOL(release_sock);
3476 
__lock_sock_fast(struct sock * sk)3477 bool __lock_sock_fast(struct sock *sk) __acquires(&sk->sk_lock.slock)
3478 {
3479 	might_sleep();
3480 	spin_lock_bh(&sk->sk_lock.slock);
3481 
3482 	if (!sock_owned_by_user_nocheck(sk)) {
3483 		/*
3484 		 * Fast path return with bottom halves disabled and
3485 		 * sock::sk_lock.slock held.
3486 		 *
3487 		 * The 'mutex' is not contended and holding
3488 		 * sock::sk_lock.slock prevents all other lockers to
3489 		 * proceed so the corresponding unlock_sock_fast() can
3490 		 * avoid the slow path of release_sock() completely and
3491 		 * just release slock.
3492 		 *
3493 		 * From a semantical POV this is equivalent to 'acquiring'
3494 		 * the 'mutex', hence the corresponding lockdep
3495 		 * mutex_release() has to happen in the fast path of
3496 		 * unlock_sock_fast().
3497 		 */
3498 		return false;
3499 	}
3500 
3501 	__lock_sock(sk);
3502 	sk->sk_lock.owned = 1;
3503 	__acquire(&sk->sk_lock.slock);
3504 	spin_unlock_bh(&sk->sk_lock.slock);
3505 	return true;
3506 }
3507 EXPORT_SYMBOL(__lock_sock_fast);
3508 
sock_gettstamp(struct socket * sock,void __user * userstamp,bool timeval,bool time32)3509 int sock_gettstamp(struct socket *sock, void __user *userstamp,
3510 		   bool timeval, bool time32)
3511 {
3512 	struct sock *sk = sock->sk;
3513 	struct timespec64 ts;
3514 
3515 	sock_enable_timestamp(sk, SOCK_TIMESTAMP);
3516 	ts = ktime_to_timespec64(sock_read_timestamp(sk));
3517 	if (ts.tv_sec == -1)
3518 		return -ENOENT;
3519 	if (ts.tv_sec == 0) {
3520 		ktime_t kt = ktime_get_real();
3521 		sock_write_timestamp(sk, kt);
3522 		ts = ktime_to_timespec64(kt);
3523 	}
3524 
3525 	if (timeval)
3526 		ts.tv_nsec /= 1000;
3527 
3528 #ifdef CONFIG_COMPAT_32BIT_TIME
3529 	if (time32)
3530 		return put_old_timespec32(&ts, userstamp);
3531 #endif
3532 #ifdef CONFIG_SPARC64
3533 	/* beware of padding in sparc64 timeval */
3534 	if (timeval && !in_compat_syscall()) {
3535 		struct __kernel_old_timeval __user tv = {
3536 			.tv_sec = ts.tv_sec,
3537 			.tv_usec = ts.tv_nsec,
3538 		};
3539 		if (copy_to_user(userstamp, &tv, sizeof(tv)))
3540 			return -EFAULT;
3541 		return 0;
3542 	}
3543 #endif
3544 	return put_timespec64(&ts, userstamp);
3545 }
3546 EXPORT_SYMBOL(sock_gettstamp);
3547 
sock_enable_timestamp(struct sock * sk,enum sock_flags flag)3548 void sock_enable_timestamp(struct sock *sk, enum sock_flags flag)
3549 {
3550 	if (!sock_flag(sk, flag)) {
3551 		unsigned long previous_flags = sk->sk_flags;
3552 
3553 		sock_set_flag(sk, flag);
3554 		/*
3555 		 * we just set one of the two flags which require net
3556 		 * time stamping, but time stamping might have been on
3557 		 * already because of the other one
3558 		 */
3559 		if (sock_needs_netstamp(sk) &&
3560 		    !(previous_flags & SK_FLAGS_TIMESTAMP))
3561 			net_enable_timestamp();
3562 	}
3563 }
3564 
sock_recv_errqueue(struct sock * sk,struct msghdr * msg,int len,int level,int type)3565 int sock_recv_errqueue(struct sock *sk, struct msghdr *msg, int len,
3566 		       int level, int type)
3567 {
3568 	struct sock_exterr_skb *serr;
3569 	struct sk_buff *skb;
3570 	int copied, err;
3571 
3572 	err = -EAGAIN;
3573 	skb = sock_dequeue_err_skb(sk);
3574 	if (skb == NULL)
3575 		goto out;
3576 
3577 	copied = skb->len;
3578 	if (copied > len) {
3579 		msg->msg_flags |= MSG_TRUNC;
3580 		copied = len;
3581 	}
3582 	err = skb_copy_datagram_msg(skb, 0, msg, copied);
3583 	if (err)
3584 		goto out_free_skb;
3585 
3586 	sock_recv_timestamp(msg, sk, skb);
3587 
3588 	serr = SKB_EXT_ERR(skb);
3589 	put_cmsg(msg, level, type, sizeof(serr->ee), &serr->ee);
3590 
3591 	msg->msg_flags |= MSG_ERRQUEUE;
3592 	err = copied;
3593 
3594 out_free_skb:
3595 	kfree_skb(skb);
3596 out:
3597 	return err;
3598 }
3599 EXPORT_SYMBOL(sock_recv_errqueue);
3600 
3601 /*
3602  *	Get a socket option on an socket.
3603  *
3604  *	FIX: POSIX 1003.1g is very ambiguous here. It states that
3605  *	asynchronous errors should be reported by getsockopt. We assume
3606  *	this means if you specify SO_ERROR (otherwise whats the point of it).
3607  */
sock_common_getsockopt(struct socket * sock,int level,int optname,char __user * optval,int __user * optlen)3608 int sock_common_getsockopt(struct socket *sock, int level, int optname,
3609 			   char __user *optval, int __user *optlen)
3610 {
3611 	struct sock *sk = sock->sk;
3612 
3613 	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
3614 	return READ_ONCE(sk->sk_prot)->getsockopt(sk, level, optname, optval, optlen);
3615 }
3616 EXPORT_SYMBOL(sock_common_getsockopt);
3617 
sock_common_recvmsg(struct socket * sock,struct msghdr * msg,size_t size,int flags)3618 int sock_common_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
3619 			int flags)
3620 {
3621 	struct sock *sk = sock->sk;
3622 	int addr_len = 0;
3623 	int err;
3624 
3625 	err = sk->sk_prot->recvmsg(sk, msg, size, flags, &addr_len);
3626 	if (err >= 0)
3627 		msg->msg_namelen = addr_len;
3628 	return err;
3629 }
3630 EXPORT_SYMBOL(sock_common_recvmsg);
3631 
3632 /*
3633  *	Set socket options on an inet socket.
3634  */
sock_common_setsockopt(struct socket * sock,int level,int optname,sockptr_t optval,unsigned int optlen)3635 int sock_common_setsockopt(struct socket *sock, int level, int optname,
3636 			   sockptr_t optval, unsigned int optlen)
3637 {
3638 	struct sock *sk = sock->sk;
3639 
3640 	/* IPV6_ADDRFORM can change sk->sk_prot under us. */
3641 	return READ_ONCE(sk->sk_prot)->setsockopt(sk, level, optname, optval, optlen);
3642 }
3643 EXPORT_SYMBOL(sock_common_setsockopt);
3644 
sk_common_release(struct sock * sk)3645 void sk_common_release(struct sock *sk)
3646 {
3647 	if (sk->sk_prot->destroy)
3648 		sk->sk_prot->destroy(sk);
3649 
3650 	/*
3651 	 * Observation: when sk_common_release is called, processes have
3652 	 * no access to socket. But net still has.
3653 	 * Step one, detach it from networking:
3654 	 *
3655 	 * A. Remove from hash tables.
3656 	 */
3657 
3658 	sk->sk_prot->unhash(sk);
3659 
3660 	/*
3661 	 * In this point socket cannot receive new packets, but it is possible
3662 	 * that some packets are in flight because some CPU runs receiver and
3663 	 * did hash table lookup before we unhashed socket. They will achieve
3664 	 * receive queue and will be purged by socket destructor.
3665 	 *
3666 	 * Also we still have packets pending on receive queue and probably,
3667 	 * our own packets waiting in device queues. sock_destroy will drain
3668 	 * receive queue, but transmitted packets will delay socket destruction
3669 	 * until the last reference will be released.
3670 	 */
3671 
3672 	sock_orphan(sk);
3673 
3674 	xfrm_sk_free_policy(sk);
3675 
3676 	sk_refcnt_debug_release(sk);
3677 
3678 	sock_put(sk);
3679 }
3680 EXPORT_SYMBOL(sk_common_release);
3681 
sk_get_meminfo(const struct sock * sk,u32 * mem)3682 void sk_get_meminfo(const struct sock *sk, u32 *mem)
3683 {
3684 	memset(mem, 0, sizeof(*mem) * SK_MEMINFO_VARS);
3685 
3686 	mem[SK_MEMINFO_RMEM_ALLOC] = sk_rmem_alloc_get(sk);
3687 	mem[SK_MEMINFO_RCVBUF] = READ_ONCE(sk->sk_rcvbuf);
3688 	mem[SK_MEMINFO_WMEM_ALLOC] = sk_wmem_alloc_get(sk);
3689 	mem[SK_MEMINFO_SNDBUF] = READ_ONCE(sk->sk_sndbuf);
3690 	mem[SK_MEMINFO_FWD_ALLOC] = sk->sk_forward_alloc;
3691 	mem[SK_MEMINFO_WMEM_QUEUED] = READ_ONCE(sk->sk_wmem_queued);
3692 	mem[SK_MEMINFO_OPTMEM] = atomic_read(&sk->sk_omem_alloc);
3693 	mem[SK_MEMINFO_BACKLOG] = READ_ONCE(sk->sk_backlog.len);
3694 	mem[SK_MEMINFO_DROPS] = atomic_read(&sk->sk_drops);
3695 }
3696 
3697 #ifdef CONFIG_PROC_FS
3698 static DECLARE_BITMAP(proto_inuse_idx, PROTO_INUSE_NR);
3699 
sock_prot_inuse_get(struct net * net,struct proto * prot)3700 int sock_prot_inuse_get(struct net *net, struct proto *prot)
3701 {
3702 	int cpu, idx = prot->inuse_idx;
3703 	int res = 0;
3704 
3705 	for_each_possible_cpu(cpu)
3706 		res += per_cpu_ptr(net->core.prot_inuse, cpu)->val[idx];
3707 
3708 	return res >= 0 ? res : 0;
3709 }
3710 EXPORT_SYMBOL_GPL(sock_prot_inuse_get);
3711 
sock_inuse_get(struct net * net)3712 int sock_inuse_get(struct net *net)
3713 {
3714 	int cpu, res = 0;
3715 
3716 	for_each_possible_cpu(cpu)
3717 		res += per_cpu_ptr(net->core.prot_inuse, cpu)->all;
3718 
3719 	return res;
3720 }
3721 
3722 EXPORT_SYMBOL_GPL(sock_inuse_get);
3723 
sock_inuse_init_net(struct net * net)3724 static int __net_init sock_inuse_init_net(struct net *net)
3725 {
3726 	net->core.prot_inuse = alloc_percpu(struct prot_inuse);
3727 	if (net->core.prot_inuse == NULL)
3728 		return -ENOMEM;
3729 	return 0;
3730 }
3731 
sock_inuse_exit_net(struct net * net)3732 static void __net_exit sock_inuse_exit_net(struct net *net)
3733 {
3734 	free_percpu(net->core.prot_inuse);
3735 }
3736 
3737 static struct pernet_operations net_inuse_ops = {
3738 	.init = sock_inuse_init_net,
3739 	.exit = sock_inuse_exit_net,
3740 };
3741 
net_inuse_init(void)3742 static __init int net_inuse_init(void)
3743 {
3744 	if (register_pernet_subsys(&net_inuse_ops))
3745 		panic("Cannot initialize net inuse counters");
3746 
3747 	return 0;
3748 }
3749 
3750 core_initcall(net_inuse_init);
3751 
assign_proto_idx(struct proto * prot)3752 static int assign_proto_idx(struct proto *prot)
3753 {
3754 	prot->inuse_idx = find_first_zero_bit(proto_inuse_idx, PROTO_INUSE_NR);
3755 
3756 	if (unlikely(prot->inuse_idx == PROTO_INUSE_NR - 1)) {
3757 		pr_err("PROTO_INUSE_NR exhausted\n");
3758 		return -ENOSPC;
3759 	}
3760 
3761 	set_bit(prot->inuse_idx, proto_inuse_idx);
3762 	return 0;
3763 }
3764 
release_proto_idx(struct proto * prot)3765 static void release_proto_idx(struct proto *prot)
3766 {
3767 	if (prot->inuse_idx != PROTO_INUSE_NR - 1)
3768 		clear_bit(prot->inuse_idx, proto_inuse_idx);
3769 }
3770 #else
assign_proto_idx(struct proto * prot)3771 static inline int assign_proto_idx(struct proto *prot)
3772 {
3773 	return 0;
3774 }
3775 
release_proto_idx(struct proto * prot)3776 static inline void release_proto_idx(struct proto *prot)
3777 {
3778 }
3779 
3780 #endif
3781 
tw_prot_cleanup(struct timewait_sock_ops * twsk_prot)3782 static void tw_prot_cleanup(struct timewait_sock_ops *twsk_prot)
3783 {
3784 	if (!twsk_prot)
3785 		return;
3786 	kfree(twsk_prot->twsk_slab_name);
3787 	twsk_prot->twsk_slab_name = NULL;
3788 	kmem_cache_destroy(twsk_prot->twsk_slab);
3789 	twsk_prot->twsk_slab = NULL;
3790 }
3791 
tw_prot_init(const struct proto * prot)3792 static int tw_prot_init(const struct proto *prot)
3793 {
3794 	struct timewait_sock_ops *twsk_prot = prot->twsk_prot;
3795 
3796 	if (!twsk_prot)
3797 		return 0;
3798 
3799 	twsk_prot->twsk_slab_name = kasprintf(GFP_KERNEL, "tw_sock_%s",
3800 					      prot->name);
3801 	if (!twsk_prot->twsk_slab_name)
3802 		return -ENOMEM;
3803 
3804 	twsk_prot->twsk_slab =
3805 		kmem_cache_create(twsk_prot->twsk_slab_name,
3806 				  twsk_prot->twsk_obj_size, 0,
3807 				  SLAB_ACCOUNT | prot->slab_flags,
3808 				  NULL);
3809 	if (!twsk_prot->twsk_slab) {
3810 		pr_crit("%s: Can't create timewait sock SLAB cache!\n",
3811 			prot->name);
3812 		return -ENOMEM;
3813 	}
3814 
3815 	return 0;
3816 }
3817 
req_prot_cleanup(struct request_sock_ops * rsk_prot)3818 static void req_prot_cleanup(struct request_sock_ops *rsk_prot)
3819 {
3820 	if (!rsk_prot)
3821 		return;
3822 	kfree(rsk_prot->slab_name);
3823 	rsk_prot->slab_name = NULL;
3824 	kmem_cache_destroy(rsk_prot->slab);
3825 	rsk_prot->slab = NULL;
3826 }
3827 
req_prot_init(const struct proto * prot)3828 static int req_prot_init(const struct proto *prot)
3829 {
3830 	struct request_sock_ops *rsk_prot = prot->rsk_prot;
3831 
3832 	if (!rsk_prot)
3833 		return 0;
3834 
3835 	rsk_prot->slab_name = kasprintf(GFP_KERNEL, "request_sock_%s",
3836 					prot->name);
3837 	if (!rsk_prot->slab_name)
3838 		return -ENOMEM;
3839 
3840 	rsk_prot->slab = kmem_cache_create(rsk_prot->slab_name,
3841 					   rsk_prot->obj_size, 0,
3842 					   SLAB_ACCOUNT | prot->slab_flags,
3843 					   NULL);
3844 
3845 	if (!rsk_prot->slab) {
3846 		pr_crit("%s: Can't create request sock SLAB cache!\n",
3847 			prot->name);
3848 		return -ENOMEM;
3849 	}
3850 	return 0;
3851 }
3852 
proto_register(struct proto * prot,int alloc_slab)3853 int proto_register(struct proto *prot, int alloc_slab)
3854 {
3855 	int ret = -ENOBUFS;
3856 
3857 	if (prot->memory_allocated && !prot->sysctl_mem) {
3858 		pr_err("%s: missing sysctl_mem\n", prot->name);
3859 		return -EINVAL;
3860 	}
3861 	if (prot->memory_allocated && !prot->per_cpu_fw_alloc) {
3862 		pr_err("%s: missing per_cpu_fw_alloc\n", prot->name);
3863 		return -EINVAL;
3864 	}
3865 	if (alloc_slab) {
3866 		prot->slab = kmem_cache_create_usercopy(prot->name,
3867 					prot->obj_size, 0,
3868 					SLAB_HWCACHE_ALIGN | SLAB_ACCOUNT |
3869 					prot->slab_flags,
3870 					prot->useroffset, prot->usersize,
3871 					NULL);
3872 
3873 		if (prot->slab == NULL) {
3874 			pr_crit("%s: Can't create sock SLAB cache!\n",
3875 				prot->name);
3876 			goto out;
3877 		}
3878 
3879 		if (req_prot_init(prot))
3880 			goto out_free_request_sock_slab;
3881 
3882 		if (tw_prot_init(prot))
3883 			goto out_free_timewait_sock_slab;
3884 	}
3885 
3886 	mutex_lock(&proto_list_mutex);
3887 	ret = assign_proto_idx(prot);
3888 	if (ret) {
3889 		mutex_unlock(&proto_list_mutex);
3890 		goto out_free_timewait_sock_slab;
3891 	}
3892 	list_add(&prot->node, &proto_list);
3893 	mutex_unlock(&proto_list_mutex);
3894 	return ret;
3895 
3896 out_free_timewait_sock_slab:
3897 	if (alloc_slab)
3898 		tw_prot_cleanup(prot->twsk_prot);
3899 out_free_request_sock_slab:
3900 	if (alloc_slab) {
3901 		req_prot_cleanup(prot->rsk_prot);
3902 
3903 		kmem_cache_destroy(prot->slab);
3904 		prot->slab = NULL;
3905 	}
3906 out:
3907 	return ret;
3908 }
3909 EXPORT_SYMBOL(proto_register);
3910 
proto_unregister(struct proto * prot)3911 void proto_unregister(struct proto *prot)
3912 {
3913 	mutex_lock(&proto_list_mutex);
3914 	release_proto_idx(prot);
3915 	list_del(&prot->node);
3916 	mutex_unlock(&proto_list_mutex);
3917 
3918 	kmem_cache_destroy(prot->slab);
3919 	prot->slab = NULL;
3920 
3921 	req_prot_cleanup(prot->rsk_prot);
3922 	tw_prot_cleanup(prot->twsk_prot);
3923 }
3924 EXPORT_SYMBOL(proto_unregister);
3925 
sock_load_diag_module(int family,int protocol)3926 int sock_load_diag_module(int family, int protocol)
3927 {
3928 	if (!protocol) {
3929 		if (!sock_is_registered(family))
3930 			return -ENOENT;
3931 
3932 		return request_module("net-pf-%d-proto-%d-type-%d", PF_NETLINK,
3933 				      NETLINK_SOCK_DIAG, family);
3934 	}
3935 
3936 #ifdef CONFIG_INET
3937 	if (family == AF_INET &&
3938 	    protocol != IPPROTO_RAW &&
3939 	    protocol < MAX_INET_PROTOS &&
3940 	    !rcu_access_pointer(inet_protos[protocol]))
3941 		return -ENOENT;
3942 #endif
3943 
3944 	return request_module("net-pf-%d-proto-%d-type-%d-%d", PF_NETLINK,
3945 			      NETLINK_SOCK_DIAG, family, protocol);
3946 }
3947 EXPORT_SYMBOL(sock_load_diag_module);
3948 
3949 #ifdef CONFIG_PROC_FS
proto_seq_start(struct seq_file * seq,loff_t * pos)3950 static void *proto_seq_start(struct seq_file *seq, loff_t *pos)
3951 	__acquires(proto_list_mutex)
3952 {
3953 	mutex_lock(&proto_list_mutex);
3954 	return seq_list_start_head(&proto_list, *pos);
3955 }
3956 
proto_seq_next(struct seq_file * seq,void * v,loff_t * pos)3957 static void *proto_seq_next(struct seq_file *seq, void *v, loff_t *pos)
3958 {
3959 	return seq_list_next(v, &proto_list, pos);
3960 }
3961 
proto_seq_stop(struct seq_file * seq,void * v)3962 static void proto_seq_stop(struct seq_file *seq, void *v)
3963 	__releases(proto_list_mutex)
3964 {
3965 	mutex_unlock(&proto_list_mutex);
3966 }
3967 
proto_method_implemented(const void * method)3968 static char proto_method_implemented(const void *method)
3969 {
3970 	return method == NULL ? 'n' : 'y';
3971 }
sock_prot_memory_allocated(struct proto * proto)3972 static long sock_prot_memory_allocated(struct proto *proto)
3973 {
3974 	return proto->memory_allocated != NULL ? proto_memory_allocated(proto) : -1L;
3975 }
3976 
sock_prot_memory_pressure(struct proto * proto)3977 static const char *sock_prot_memory_pressure(struct proto *proto)
3978 {
3979 	return proto->memory_pressure != NULL ?
3980 	proto_memory_pressure(proto) ? "yes" : "no" : "NI";
3981 }
3982 
proto_seq_printf(struct seq_file * seq,struct proto * proto)3983 static void proto_seq_printf(struct seq_file *seq, struct proto *proto)
3984 {
3985 
3986 	seq_printf(seq, "%-9s %4u %6d  %6ld   %-3s %6u   %-3s  %-10s "
3987 			"%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n",
3988 		   proto->name,
3989 		   proto->obj_size,
3990 		   sock_prot_inuse_get(seq_file_net(seq), proto),
3991 		   sock_prot_memory_allocated(proto),
3992 		   sock_prot_memory_pressure(proto),
3993 		   proto->max_header,
3994 		   proto->slab == NULL ? "no" : "yes",
3995 		   module_name(proto->owner),
3996 		   proto_method_implemented(proto->close),
3997 		   proto_method_implemented(proto->connect),
3998 		   proto_method_implemented(proto->disconnect),
3999 		   proto_method_implemented(proto->accept),
4000 		   proto_method_implemented(proto->ioctl),
4001 		   proto_method_implemented(proto->init),
4002 		   proto_method_implemented(proto->destroy),
4003 		   proto_method_implemented(proto->shutdown),
4004 		   proto_method_implemented(proto->setsockopt),
4005 		   proto_method_implemented(proto->getsockopt),
4006 		   proto_method_implemented(proto->sendmsg),
4007 		   proto_method_implemented(proto->recvmsg),
4008 		   proto_method_implemented(proto->sendpage),
4009 		   proto_method_implemented(proto->bind),
4010 		   proto_method_implemented(proto->backlog_rcv),
4011 		   proto_method_implemented(proto->hash),
4012 		   proto_method_implemented(proto->unhash),
4013 		   proto_method_implemented(proto->get_port),
4014 		   proto_method_implemented(proto->enter_memory_pressure));
4015 }
4016 
proto_seq_show(struct seq_file * seq,void * v)4017 static int proto_seq_show(struct seq_file *seq, void *v)
4018 {
4019 	if (v == &proto_list)
4020 		seq_printf(seq, "%-9s %-4s %-8s %-6s %-5s %-7s %-4s %-10s %s",
4021 			   "protocol",
4022 			   "size",
4023 			   "sockets",
4024 			   "memory",
4025 			   "press",
4026 			   "maxhdr",
4027 			   "slab",
4028 			   "module",
4029 			   "cl co di ac io in de sh ss gs se re sp bi br ha uh gp em\n");
4030 	else
4031 		proto_seq_printf(seq, list_entry(v, struct proto, node));
4032 	return 0;
4033 }
4034 
4035 static const struct seq_operations proto_seq_ops = {
4036 	.start  = proto_seq_start,
4037 	.next   = proto_seq_next,
4038 	.stop   = proto_seq_stop,
4039 	.show   = proto_seq_show,
4040 };
4041 
proto_init_net(struct net * net)4042 static __net_init int proto_init_net(struct net *net)
4043 {
4044 	if (!proc_create_net("protocols", 0444, net->proc_net, &proto_seq_ops,
4045 			sizeof(struct seq_net_private)))
4046 		return -ENOMEM;
4047 
4048 	return 0;
4049 }
4050 
proto_exit_net(struct net * net)4051 static __net_exit void proto_exit_net(struct net *net)
4052 {
4053 	remove_proc_entry("protocols", net->proc_net);
4054 }
4055 
4056 
4057 static __net_initdata struct pernet_operations proto_net_ops = {
4058 	.init = proto_init_net,
4059 	.exit = proto_exit_net,
4060 };
4061 
proto_init(void)4062 static int __init proto_init(void)
4063 {
4064 	return register_pernet_subsys(&proto_net_ops);
4065 }
4066 
4067 subsys_initcall(proto_init);
4068 
4069 #endif /* PROC_FS */
4070 
4071 #ifdef CONFIG_NET_RX_BUSY_POLL
sk_busy_loop_end(void * p,unsigned long start_time)4072 bool sk_busy_loop_end(void *p, unsigned long start_time)
4073 {
4074 	struct sock *sk = p;
4075 
4076 	return !skb_queue_empty_lockless(&sk->sk_receive_queue) ||
4077 	       sk_busy_loop_timeout(sk, start_time);
4078 }
4079 EXPORT_SYMBOL(sk_busy_loop_end);
4080 #endif /* CONFIG_NET_RX_BUSY_POLL */
4081 
sock_bind_add(struct sock * sk,struct sockaddr * addr,int addr_len)4082 int sock_bind_add(struct sock *sk, struct sockaddr *addr, int addr_len)
4083 {
4084 	if (!sk->sk_prot->bind_add)
4085 		return -EOPNOTSUPP;
4086 	return sk->sk_prot->bind_add(sk, addr, addr_len);
4087 }
4088 EXPORT_SYMBOL(sock_bind_add);
4089