1 /*
2  * Copyright (c) 2016-2017, Mellanox Technologies. All rights reserved.
3  * Copyright (c) 2016-2017, Dave Watson <davejwatson@fb.com>. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *      - Redistributions of source code must retain the above
16  *        copyright notice, this list of conditions and the following
17  *        disclaimer.
18  *
19  *      - Redistributions in binary form must reproduce the above
20  *        copyright notice, this list of conditions and the following
21  *        disclaimer in the documentation and/or other materials
22  *        provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 
34 #include <linux/module.h>
35 
36 #include <net/tcp.h>
37 #include <net/inet_common.h>
38 #include <linux/highmem.h>
39 #include <linux/netdevice.h>
40 #include <linux/sched/signal.h>
41 #include <linux/inetdevice.h>
42 #include <linux/inet_diag.h>
43 
44 #include <net/snmp.h>
45 #include <net/tls.h>
46 #include <net/tls_toe.h>
47 
48 #include "tls.h"
49 
50 MODULE_AUTHOR("Mellanox Technologies");
51 MODULE_DESCRIPTION("Transport Layer Security Support");
52 MODULE_LICENSE("Dual BSD/GPL");
53 MODULE_ALIAS_TCP_ULP("tls");
54 
55 enum {
56 	TLSV4,
57 	TLSV6,
58 	TLS_NUM_PROTS,
59 };
60 
61 #define CIPHER_SIZE_DESC(cipher) [cipher] = { \
62 	.iv = cipher ## _IV_SIZE, \
63 	.key = cipher ## _KEY_SIZE, \
64 	.salt = cipher ## _SALT_SIZE, \
65 	.tag = cipher ## _TAG_SIZE, \
66 	.rec_seq = cipher ## _REC_SEQ_SIZE, \
67 }
68 
69 const struct tls_cipher_size_desc tls_cipher_size_desc[] = {
70 	CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_128),
71 	CIPHER_SIZE_DESC(TLS_CIPHER_AES_GCM_256),
72 	CIPHER_SIZE_DESC(TLS_CIPHER_AES_CCM_128),
73 	CIPHER_SIZE_DESC(TLS_CIPHER_CHACHA20_POLY1305),
74 	CIPHER_SIZE_DESC(TLS_CIPHER_SM4_GCM),
75 	CIPHER_SIZE_DESC(TLS_CIPHER_SM4_CCM),
76 };
77 
78 static const struct proto *saved_tcpv6_prot;
79 static DEFINE_MUTEX(tcpv6_prot_mutex);
80 static const struct proto *saved_tcpv4_prot;
81 static DEFINE_MUTEX(tcpv4_prot_mutex);
82 static struct proto tls_prots[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
83 static struct proto_ops tls_proto_ops[TLS_NUM_PROTS][TLS_NUM_CONFIG][TLS_NUM_CONFIG];
84 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
85 			 const struct proto *base);
86 
update_sk_prot(struct sock * sk,struct tls_context * ctx)87 void update_sk_prot(struct sock *sk, struct tls_context *ctx)
88 {
89 	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
90 
91 	WRITE_ONCE(sk->sk_prot,
92 		   &tls_prots[ip_ver][ctx->tx_conf][ctx->rx_conf]);
93 	WRITE_ONCE(sk->sk_socket->ops,
94 		   &tls_proto_ops[ip_ver][ctx->tx_conf][ctx->rx_conf]);
95 }
96 
wait_on_pending_writer(struct sock * sk,long * timeo)97 int wait_on_pending_writer(struct sock *sk, long *timeo)
98 {
99 	int rc = 0;
100 	DEFINE_WAIT_FUNC(wait, woken_wake_function);
101 
102 	add_wait_queue(sk_sleep(sk), &wait);
103 	while (1) {
104 		if (!*timeo) {
105 			rc = -EAGAIN;
106 			break;
107 		}
108 
109 		if (signal_pending(current)) {
110 			rc = sock_intr_errno(*timeo);
111 			break;
112 		}
113 
114 		if (sk_wait_event(sk, timeo, !sk->sk_write_pending, &wait))
115 			break;
116 	}
117 	remove_wait_queue(sk_sleep(sk), &wait);
118 	return rc;
119 }
120 
tls_push_sg(struct sock * sk,struct tls_context * ctx,struct scatterlist * sg,u16 first_offset,int flags)121 int tls_push_sg(struct sock *sk,
122 		struct tls_context *ctx,
123 		struct scatterlist *sg,
124 		u16 first_offset,
125 		int flags)
126 {
127 	int sendpage_flags = flags | MSG_SENDPAGE_NOTLAST;
128 	int ret = 0;
129 	struct page *p;
130 	size_t size;
131 	int offset = first_offset;
132 
133 	size = sg->length - offset;
134 	offset += sg->offset;
135 
136 	ctx->in_tcp_sendpages = true;
137 	while (1) {
138 		if (sg_is_last(sg))
139 			sendpage_flags = flags;
140 
141 		/* is sending application-limited? */
142 		tcp_rate_check_app_limited(sk);
143 		p = sg_page(sg);
144 retry:
145 		ret = do_tcp_sendpages(sk, p, offset, size, sendpage_flags);
146 
147 		if (ret != size) {
148 			if (ret > 0) {
149 				offset += ret;
150 				size -= ret;
151 				goto retry;
152 			}
153 
154 			offset -= sg->offset;
155 			ctx->partially_sent_offset = offset;
156 			ctx->partially_sent_record = (void *)sg;
157 			ctx->in_tcp_sendpages = false;
158 			return ret;
159 		}
160 
161 		put_page(p);
162 		sk_mem_uncharge(sk, sg->length);
163 		sg = sg_next(sg);
164 		if (!sg)
165 			break;
166 
167 		offset = sg->offset;
168 		size = sg->length;
169 	}
170 
171 	ctx->in_tcp_sendpages = false;
172 
173 	return 0;
174 }
175 
tls_handle_open_record(struct sock * sk,int flags)176 static int tls_handle_open_record(struct sock *sk, int flags)
177 {
178 	struct tls_context *ctx = tls_get_ctx(sk);
179 
180 	if (tls_is_pending_open_record(ctx))
181 		return ctx->push_pending_record(sk, flags);
182 
183 	return 0;
184 }
185 
tls_process_cmsg(struct sock * sk,struct msghdr * msg,unsigned char * record_type)186 int tls_process_cmsg(struct sock *sk, struct msghdr *msg,
187 		     unsigned char *record_type)
188 {
189 	struct cmsghdr *cmsg;
190 	int rc = -EINVAL;
191 
192 	for_each_cmsghdr(cmsg, msg) {
193 		if (!CMSG_OK(msg, cmsg))
194 			return -EINVAL;
195 		if (cmsg->cmsg_level != SOL_TLS)
196 			continue;
197 
198 		switch (cmsg->cmsg_type) {
199 		case TLS_SET_RECORD_TYPE:
200 			if (cmsg->cmsg_len < CMSG_LEN(sizeof(*record_type)))
201 				return -EINVAL;
202 
203 			if (msg->msg_flags & MSG_MORE)
204 				return -EINVAL;
205 
206 			rc = tls_handle_open_record(sk, msg->msg_flags);
207 			if (rc)
208 				return rc;
209 
210 			*record_type = *(unsigned char *)CMSG_DATA(cmsg);
211 			rc = 0;
212 			break;
213 		default:
214 			return -EINVAL;
215 		}
216 	}
217 
218 	return rc;
219 }
220 
tls_push_partial_record(struct sock * sk,struct tls_context * ctx,int flags)221 int tls_push_partial_record(struct sock *sk, struct tls_context *ctx,
222 			    int flags)
223 {
224 	struct scatterlist *sg;
225 	u16 offset;
226 
227 	sg = ctx->partially_sent_record;
228 	offset = ctx->partially_sent_offset;
229 
230 	ctx->partially_sent_record = NULL;
231 	return tls_push_sg(sk, ctx, sg, offset, flags);
232 }
233 
tls_free_partial_record(struct sock * sk,struct tls_context * ctx)234 void tls_free_partial_record(struct sock *sk, struct tls_context *ctx)
235 {
236 	struct scatterlist *sg;
237 
238 	for (sg = ctx->partially_sent_record; sg; sg = sg_next(sg)) {
239 		put_page(sg_page(sg));
240 		sk_mem_uncharge(sk, sg->length);
241 	}
242 	ctx->partially_sent_record = NULL;
243 }
244 
tls_write_space(struct sock * sk)245 static void tls_write_space(struct sock *sk)
246 {
247 	struct tls_context *ctx = tls_get_ctx(sk);
248 
249 	/* If in_tcp_sendpages call lower protocol write space handler
250 	 * to ensure we wake up any waiting operations there. For example
251 	 * if do_tcp_sendpages where to call sk_wait_event.
252 	 */
253 	if (ctx->in_tcp_sendpages) {
254 		ctx->sk_write_space(sk);
255 		return;
256 	}
257 
258 #ifdef CONFIG_TLS_DEVICE
259 	if (ctx->tx_conf == TLS_HW)
260 		tls_device_write_space(sk, ctx);
261 	else
262 #endif
263 		tls_sw_write_space(sk, ctx);
264 
265 	ctx->sk_write_space(sk);
266 }
267 
268 /**
269  * tls_ctx_free() - free TLS ULP context
270  * @sk:  socket to with @ctx is attached
271  * @ctx: TLS context structure
272  *
273  * Free TLS context. If @sk is %NULL caller guarantees that the socket
274  * to which @ctx was attached has no outstanding references.
275  */
tls_ctx_free(struct sock * sk,struct tls_context * ctx)276 void tls_ctx_free(struct sock *sk, struct tls_context *ctx)
277 {
278 	if (!ctx)
279 		return;
280 
281 	memzero_explicit(&ctx->crypto_send, sizeof(ctx->crypto_send));
282 	memzero_explicit(&ctx->crypto_recv, sizeof(ctx->crypto_recv));
283 	mutex_destroy(&ctx->tx_lock);
284 
285 	if (sk)
286 		kfree_rcu(ctx, rcu);
287 	else
288 		kfree(ctx);
289 }
290 
tls_sk_proto_cleanup(struct sock * sk,struct tls_context * ctx,long timeo)291 static void tls_sk_proto_cleanup(struct sock *sk,
292 				 struct tls_context *ctx, long timeo)
293 {
294 	if (unlikely(sk->sk_write_pending) &&
295 	    !wait_on_pending_writer(sk, &timeo))
296 		tls_handle_open_record(sk, 0);
297 
298 	/* We need these for tls_sw_fallback handling of other packets */
299 	if (ctx->tx_conf == TLS_SW) {
300 		kfree(ctx->tx.rec_seq);
301 		kfree(ctx->tx.iv);
302 		tls_sw_release_resources_tx(sk);
303 		TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
304 	} else if (ctx->tx_conf == TLS_HW) {
305 		tls_device_free_resources_tx(sk);
306 		TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
307 	}
308 
309 	if (ctx->rx_conf == TLS_SW) {
310 		tls_sw_release_resources_rx(sk);
311 		TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
312 	} else if (ctx->rx_conf == TLS_HW) {
313 		tls_device_offload_cleanup_rx(sk);
314 		TLS_DEC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
315 	}
316 }
317 
tls_sk_proto_close(struct sock * sk,long timeout)318 static void tls_sk_proto_close(struct sock *sk, long timeout)
319 {
320 	struct inet_connection_sock *icsk = inet_csk(sk);
321 	struct tls_context *ctx = tls_get_ctx(sk);
322 	long timeo = sock_sndtimeo(sk, 0);
323 	bool free_ctx;
324 
325 	if (ctx->tx_conf == TLS_SW)
326 		tls_sw_cancel_work_tx(ctx);
327 
328 	lock_sock(sk);
329 	free_ctx = ctx->tx_conf != TLS_HW && ctx->rx_conf != TLS_HW;
330 
331 	if (ctx->tx_conf != TLS_BASE || ctx->rx_conf != TLS_BASE)
332 		tls_sk_proto_cleanup(sk, ctx, timeo);
333 
334 	write_lock_bh(&sk->sk_callback_lock);
335 	if (free_ctx)
336 		rcu_assign_pointer(icsk->icsk_ulp_data, NULL);
337 	WRITE_ONCE(sk->sk_prot, ctx->sk_proto);
338 	if (sk->sk_write_space == tls_write_space)
339 		sk->sk_write_space = ctx->sk_write_space;
340 	write_unlock_bh(&sk->sk_callback_lock);
341 	release_sock(sk);
342 	if (ctx->tx_conf == TLS_SW)
343 		tls_sw_free_ctx_tx(ctx);
344 	if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW)
345 		tls_sw_strparser_done(ctx);
346 	if (ctx->rx_conf == TLS_SW)
347 		tls_sw_free_ctx_rx(ctx);
348 	ctx->sk_proto->close(sk, timeout);
349 
350 	if (free_ctx)
351 		tls_ctx_free(sk, ctx);
352 }
353 
do_tls_getsockopt_conf(struct sock * sk,char __user * optval,int __user * optlen,int tx)354 static int do_tls_getsockopt_conf(struct sock *sk, char __user *optval,
355 				  int __user *optlen, int tx)
356 {
357 	int rc = 0;
358 	struct tls_context *ctx = tls_get_ctx(sk);
359 	struct tls_crypto_info *crypto_info;
360 	struct cipher_context *cctx;
361 	int len;
362 
363 	if (get_user(len, optlen))
364 		return -EFAULT;
365 
366 	if (!optval || (len < sizeof(*crypto_info))) {
367 		rc = -EINVAL;
368 		goto out;
369 	}
370 
371 	if (!ctx) {
372 		rc = -EBUSY;
373 		goto out;
374 	}
375 
376 	/* get user crypto info */
377 	if (tx) {
378 		crypto_info = &ctx->crypto_send.info;
379 		cctx = &ctx->tx;
380 	} else {
381 		crypto_info = &ctx->crypto_recv.info;
382 		cctx = &ctx->rx;
383 	}
384 
385 	if (!TLS_CRYPTO_INFO_READY(crypto_info)) {
386 		rc = -EBUSY;
387 		goto out;
388 	}
389 
390 	if (len == sizeof(*crypto_info)) {
391 		if (copy_to_user(optval, crypto_info, sizeof(*crypto_info)))
392 			rc = -EFAULT;
393 		goto out;
394 	}
395 
396 	switch (crypto_info->cipher_type) {
397 	case TLS_CIPHER_AES_GCM_128: {
398 		struct tls12_crypto_info_aes_gcm_128 *
399 		  crypto_info_aes_gcm_128 =
400 		  container_of(crypto_info,
401 			       struct tls12_crypto_info_aes_gcm_128,
402 			       info);
403 
404 		if (len != sizeof(*crypto_info_aes_gcm_128)) {
405 			rc = -EINVAL;
406 			goto out;
407 		}
408 		lock_sock(sk);
409 		memcpy(crypto_info_aes_gcm_128->iv,
410 		       cctx->iv + TLS_CIPHER_AES_GCM_128_SALT_SIZE,
411 		       TLS_CIPHER_AES_GCM_128_IV_SIZE);
412 		memcpy(crypto_info_aes_gcm_128->rec_seq, cctx->rec_seq,
413 		       TLS_CIPHER_AES_GCM_128_REC_SEQ_SIZE);
414 		release_sock(sk);
415 		if (copy_to_user(optval,
416 				 crypto_info_aes_gcm_128,
417 				 sizeof(*crypto_info_aes_gcm_128)))
418 			rc = -EFAULT;
419 		break;
420 	}
421 	case TLS_CIPHER_AES_GCM_256: {
422 		struct tls12_crypto_info_aes_gcm_256 *
423 		  crypto_info_aes_gcm_256 =
424 		  container_of(crypto_info,
425 			       struct tls12_crypto_info_aes_gcm_256,
426 			       info);
427 
428 		if (len != sizeof(*crypto_info_aes_gcm_256)) {
429 			rc = -EINVAL;
430 			goto out;
431 		}
432 		lock_sock(sk);
433 		memcpy(crypto_info_aes_gcm_256->iv,
434 		       cctx->iv + TLS_CIPHER_AES_GCM_256_SALT_SIZE,
435 		       TLS_CIPHER_AES_GCM_256_IV_SIZE);
436 		memcpy(crypto_info_aes_gcm_256->rec_seq, cctx->rec_seq,
437 		       TLS_CIPHER_AES_GCM_256_REC_SEQ_SIZE);
438 		release_sock(sk);
439 		if (copy_to_user(optval,
440 				 crypto_info_aes_gcm_256,
441 				 sizeof(*crypto_info_aes_gcm_256)))
442 			rc = -EFAULT;
443 		break;
444 	}
445 	case TLS_CIPHER_AES_CCM_128: {
446 		struct tls12_crypto_info_aes_ccm_128 *aes_ccm_128 =
447 			container_of(crypto_info,
448 				struct tls12_crypto_info_aes_ccm_128, info);
449 
450 		if (len != sizeof(*aes_ccm_128)) {
451 			rc = -EINVAL;
452 			goto out;
453 		}
454 		lock_sock(sk);
455 		memcpy(aes_ccm_128->iv,
456 		       cctx->iv + TLS_CIPHER_AES_CCM_128_SALT_SIZE,
457 		       TLS_CIPHER_AES_CCM_128_IV_SIZE);
458 		memcpy(aes_ccm_128->rec_seq, cctx->rec_seq,
459 		       TLS_CIPHER_AES_CCM_128_REC_SEQ_SIZE);
460 		release_sock(sk);
461 		if (copy_to_user(optval, aes_ccm_128, sizeof(*aes_ccm_128)))
462 			rc = -EFAULT;
463 		break;
464 	}
465 	case TLS_CIPHER_CHACHA20_POLY1305: {
466 		struct tls12_crypto_info_chacha20_poly1305 *chacha20_poly1305 =
467 			container_of(crypto_info,
468 				struct tls12_crypto_info_chacha20_poly1305,
469 				info);
470 
471 		if (len != sizeof(*chacha20_poly1305)) {
472 			rc = -EINVAL;
473 			goto out;
474 		}
475 		lock_sock(sk);
476 		memcpy(chacha20_poly1305->iv,
477 		       cctx->iv + TLS_CIPHER_CHACHA20_POLY1305_SALT_SIZE,
478 		       TLS_CIPHER_CHACHA20_POLY1305_IV_SIZE);
479 		memcpy(chacha20_poly1305->rec_seq, cctx->rec_seq,
480 		       TLS_CIPHER_CHACHA20_POLY1305_REC_SEQ_SIZE);
481 		release_sock(sk);
482 		if (copy_to_user(optval, chacha20_poly1305,
483 				sizeof(*chacha20_poly1305)))
484 			rc = -EFAULT;
485 		break;
486 	}
487 	case TLS_CIPHER_SM4_GCM: {
488 		struct tls12_crypto_info_sm4_gcm *sm4_gcm_info =
489 			container_of(crypto_info,
490 				struct tls12_crypto_info_sm4_gcm, info);
491 
492 		if (len != sizeof(*sm4_gcm_info)) {
493 			rc = -EINVAL;
494 			goto out;
495 		}
496 		lock_sock(sk);
497 		memcpy(sm4_gcm_info->iv,
498 		       cctx->iv + TLS_CIPHER_SM4_GCM_SALT_SIZE,
499 		       TLS_CIPHER_SM4_GCM_IV_SIZE);
500 		memcpy(sm4_gcm_info->rec_seq, cctx->rec_seq,
501 		       TLS_CIPHER_SM4_GCM_REC_SEQ_SIZE);
502 		release_sock(sk);
503 		if (copy_to_user(optval, sm4_gcm_info, sizeof(*sm4_gcm_info)))
504 			rc = -EFAULT;
505 		break;
506 	}
507 	case TLS_CIPHER_SM4_CCM: {
508 		struct tls12_crypto_info_sm4_ccm *sm4_ccm_info =
509 			container_of(crypto_info,
510 				struct tls12_crypto_info_sm4_ccm, info);
511 
512 		if (len != sizeof(*sm4_ccm_info)) {
513 			rc = -EINVAL;
514 			goto out;
515 		}
516 		lock_sock(sk);
517 		memcpy(sm4_ccm_info->iv,
518 		       cctx->iv + TLS_CIPHER_SM4_CCM_SALT_SIZE,
519 		       TLS_CIPHER_SM4_CCM_IV_SIZE);
520 		memcpy(sm4_ccm_info->rec_seq, cctx->rec_seq,
521 		       TLS_CIPHER_SM4_CCM_REC_SEQ_SIZE);
522 		release_sock(sk);
523 		if (copy_to_user(optval, sm4_ccm_info, sizeof(*sm4_ccm_info)))
524 			rc = -EFAULT;
525 		break;
526 	}
527 	case TLS_CIPHER_ARIA_GCM_128: {
528 		struct tls12_crypto_info_aria_gcm_128 *
529 		  crypto_info_aria_gcm_128 =
530 		  container_of(crypto_info,
531 			       struct tls12_crypto_info_aria_gcm_128,
532 			       info);
533 
534 		if (len != sizeof(*crypto_info_aria_gcm_128)) {
535 			rc = -EINVAL;
536 			goto out;
537 		}
538 		lock_sock(sk);
539 		memcpy(crypto_info_aria_gcm_128->iv,
540 		       cctx->iv + TLS_CIPHER_ARIA_GCM_128_SALT_SIZE,
541 		       TLS_CIPHER_ARIA_GCM_128_IV_SIZE);
542 		memcpy(crypto_info_aria_gcm_128->rec_seq, cctx->rec_seq,
543 		       TLS_CIPHER_ARIA_GCM_128_REC_SEQ_SIZE);
544 		release_sock(sk);
545 		if (copy_to_user(optval,
546 				 crypto_info_aria_gcm_128,
547 				 sizeof(*crypto_info_aria_gcm_128)))
548 			rc = -EFAULT;
549 		break;
550 	}
551 	case TLS_CIPHER_ARIA_GCM_256: {
552 		struct tls12_crypto_info_aria_gcm_256 *
553 		  crypto_info_aria_gcm_256 =
554 		  container_of(crypto_info,
555 			       struct tls12_crypto_info_aria_gcm_256,
556 			       info);
557 
558 		if (len != sizeof(*crypto_info_aria_gcm_256)) {
559 			rc = -EINVAL;
560 			goto out;
561 		}
562 		lock_sock(sk);
563 		memcpy(crypto_info_aria_gcm_256->iv,
564 		       cctx->iv + TLS_CIPHER_ARIA_GCM_256_SALT_SIZE,
565 		       TLS_CIPHER_ARIA_GCM_256_IV_SIZE);
566 		memcpy(crypto_info_aria_gcm_256->rec_seq, cctx->rec_seq,
567 		       TLS_CIPHER_ARIA_GCM_256_REC_SEQ_SIZE);
568 		release_sock(sk);
569 		if (copy_to_user(optval,
570 				 crypto_info_aria_gcm_256,
571 				 sizeof(*crypto_info_aria_gcm_256)))
572 			rc = -EFAULT;
573 		break;
574 	}
575 	default:
576 		rc = -EINVAL;
577 	}
578 
579 out:
580 	return rc;
581 }
582 
do_tls_getsockopt_tx_zc(struct sock * sk,char __user * optval,int __user * optlen)583 static int do_tls_getsockopt_tx_zc(struct sock *sk, char __user *optval,
584 				   int __user *optlen)
585 {
586 	struct tls_context *ctx = tls_get_ctx(sk);
587 	unsigned int value;
588 	int len;
589 
590 	if (get_user(len, optlen))
591 		return -EFAULT;
592 
593 	if (len != sizeof(value))
594 		return -EINVAL;
595 
596 	value = ctx->zerocopy_sendfile;
597 	if (copy_to_user(optval, &value, sizeof(value)))
598 		return -EFAULT;
599 
600 	return 0;
601 }
602 
do_tls_getsockopt_no_pad(struct sock * sk,char __user * optval,int __user * optlen)603 static int do_tls_getsockopt_no_pad(struct sock *sk, char __user *optval,
604 				    int __user *optlen)
605 {
606 	struct tls_context *ctx = tls_get_ctx(sk);
607 	int value, len;
608 
609 	if (ctx->prot_info.version != TLS_1_3_VERSION)
610 		return -EINVAL;
611 
612 	if (get_user(len, optlen))
613 		return -EFAULT;
614 	if (len < sizeof(value))
615 		return -EINVAL;
616 
617 	lock_sock(sk);
618 	value = -EINVAL;
619 	if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW)
620 		value = ctx->rx_no_pad;
621 	release_sock(sk);
622 	if (value < 0)
623 		return value;
624 
625 	if (put_user(sizeof(value), optlen))
626 		return -EFAULT;
627 	if (copy_to_user(optval, &value, sizeof(value)))
628 		return -EFAULT;
629 
630 	return 0;
631 }
632 
do_tls_getsockopt(struct sock * sk,int optname,char __user * optval,int __user * optlen)633 static int do_tls_getsockopt(struct sock *sk, int optname,
634 			     char __user *optval, int __user *optlen)
635 {
636 	int rc = 0;
637 
638 	switch (optname) {
639 	case TLS_TX:
640 	case TLS_RX:
641 		rc = do_tls_getsockopt_conf(sk, optval, optlen,
642 					    optname == TLS_TX);
643 		break;
644 	case TLS_TX_ZEROCOPY_RO:
645 		rc = do_tls_getsockopt_tx_zc(sk, optval, optlen);
646 		break;
647 	case TLS_RX_EXPECT_NO_PAD:
648 		rc = do_tls_getsockopt_no_pad(sk, optval, optlen);
649 		break;
650 	default:
651 		rc = -ENOPROTOOPT;
652 		break;
653 	}
654 	return rc;
655 }
656 
tls_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)657 static int tls_getsockopt(struct sock *sk, int level, int optname,
658 			  char __user *optval, int __user *optlen)
659 {
660 	struct tls_context *ctx = tls_get_ctx(sk);
661 
662 	if (level != SOL_TLS)
663 		return ctx->sk_proto->getsockopt(sk, level,
664 						 optname, optval, optlen);
665 
666 	return do_tls_getsockopt(sk, optname, optval, optlen);
667 }
668 
do_tls_setsockopt_conf(struct sock * sk,sockptr_t optval,unsigned int optlen,int tx)669 static int do_tls_setsockopt_conf(struct sock *sk, sockptr_t optval,
670 				  unsigned int optlen, int tx)
671 {
672 	struct tls_crypto_info *crypto_info;
673 	struct tls_crypto_info *alt_crypto_info;
674 	struct tls_context *ctx = tls_get_ctx(sk);
675 	size_t optsize;
676 	int rc = 0;
677 	int conf;
678 
679 	if (sockptr_is_null(optval) || (optlen < sizeof(*crypto_info)))
680 		return -EINVAL;
681 
682 	if (tx) {
683 		crypto_info = &ctx->crypto_send.info;
684 		alt_crypto_info = &ctx->crypto_recv.info;
685 	} else {
686 		crypto_info = &ctx->crypto_recv.info;
687 		alt_crypto_info = &ctx->crypto_send.info;
688 	}
689 
690 	/* Currently we don't support set crypto info more than one time */
691 	if (TLS_CRYPTO_INFO_READY(crypto_info))
692 		return -EBUSY;
693 
694 	rc = copy_from_sockptr(crypto_info, optval, sizeof(*crypto_info));
695 	if (rc) {
696 		rc = -EFAULT;
697 		goto err_crypto_info;
698 	}
699 
700 	/* check version */
701 	if (crypto_info->version != TLS_1_2_VERSION &&
702 	    crypto_info->version != TLS_1_3_VERSION) {
703 		rc = -EINVAL;
704 		goto err_crypto_info;
705 	}
706 
707 	/* Ensure that TLS version and ciphers are same in both directions */
708 	if (TLS_CRYPTO_INFO_READY(alt_crypto_info)) {
709 		if (alt_crypto_info->version != crypto_info->version ||
710 		    alt_crypto_info->cipher_type != crypto_info->cipher_type) {
711 			rc = -EINVAL;
712 			goto err_crypto_info;
713 		}
714 	}
715 
716 	switch (crypto_info->cipher_type) {
717 	case TLS_CIPHER_AES_GCM_128:
718 		optsize = sizeof(struct tls12_crypto_info_aes_gcm_128);
719 		break;
720 	case TLS_CIPHER_AES_GCM_256: {
721 		optsize = sizeof(struct tls12_crypto_info_aes_gcm_256);
722 		break;
723 	}
724 	case TLS_CIPHER_AES_CCM_128:
725 		optsize = sizeof(struct tls12_crypto_info_aes_ccm_128);
726 		break;
727 	case TLS_CIPHER_CHACHA20_POLY1305:
728 		optsize = sizeof(struct tls12_crypto_info_chacha20_poly1305);
729 		break;
730 	case TLS_CIPHER_SM4_GCM:
731 		optsize = sizeof(struct tls12_crypto_info_sm4_gcm);
732 		break;
733 	case TLS_CIPHER_SM4_CCM:
734 		optsize = sizeof(struct tls12_crypto_info_sm4_ccm);
735 		break;
736 	case TLS_CIPHER_ARIA_GCM_128:
737 		if (crypto_info->version != TLS_1_2_VERSION) {
738 			rc = -EINVAL;
739 			goto err_crypto_info;
740 		}
741 		optsize = sizeof(struct tls12_crypto_info_aria_gcm_128);
742 		break;
743 	case TLS_CIPHER_ARIA_GCM_256:
744 		if (crypto_info->version != TLS_1_2_VERSION) {
745 			rc = -EINVAL;
746 			goto err_crypto_info;
747 		}
748 		optsize = sizeof(struct tls12_crypto_info_aria_gcm_256);
749 		break;
750 	default:
751 		rc = -EINVAL;
752 		goto err_crypto_info;
753 	}
754 
755 	if (optlen != optsize) {
756 		rc = -EINVAL;
757 		goto err_crypto_info;
758 	}
759 
760 	rc = copy_from_sockptr_offset(crypto_info + 1, optval,
761 				      sizeof(*crypto_info),
762 				      optlen - sizeof(*crypto_info));
763 	if (rc) {
764 		rc = -EFAULT;
765 		goto err_crypto_info;
766 	}
767 
768 	if (tx) {
769 		rc = tls_set_device_offload(sk, ctx);
770 		conf = TLS_HW;
771 		if (!rc) {
772 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXDEVICE);
773 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXDEVICE);
774 		} else {
775 			rc = tls_set_sw_offload(sk, ctx, 1);
776 			if (rc)
777 				goto err_crypto_info;
778 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSTXSW);
779 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRTXSW);
780 			conf = TLS_SW;
781 		}
782 	} else {
783 		rc = tls_set_device_offload_rx(sk, ctx);
784 		conf = TLS_HW;
785 		if (!rc) {
786 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXDEVICE);
787 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXDEVICE);
788 		} else {
789 			rc = tls_set_sw_offload(sk, ctx, 0);
790 			if (rc)
791 				goto err_crypto_info;
792 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSRXSW);
793 			TLS_INC_STATS(sock_net(sk), LINUX_MIB_TLSCURRRXSW);
794 			conf = TLS_SW;
795 		}
796 		tls_sw_strparser_arm(sk, ctx);
797 	}
798 
799 	if (tx)
800 		ctx->tx_conf = conf;
801 	else
802 		ctx->rx_conf = conf;
803 	update_sk_prot(sk, ctx);
804 	if (tx) {
805 		ctx->sk_write_space = sk->sk_write_space;
806 		sk->sk_write_space = tls_write_space;
807 	} else {
808 		struct tls_sw_context_rx *rx_ctx = tls_sw_ctx_rx(ctx);
809 
810 		tls_strp_check_rcv(&rx_ctx->strp);
811 	}
812 	return 0;
813 
814 err_crypto_info:
815 	memzero_explicit(crypto_info, sizeof(union tls_crypto_context));
816 	return rc;
817 }
818 
do_tls_setsockopt_tx_zc(struct sock * sk,sockptr_t optval,unsigned int optlen)819 static int do_tls_setsockopt_tx_zc(struct sock *sk, sockptr_t optval,
820 				   unsigned int optlen)
821 {
822 	struct tls_context *ctx = tls_get_ctx(sk);
823 	unsigned int value;
824 
825 	if (sockptr_is_null(optval) || optlen != sizeof(value))
826 		return -EINVAL;
827 
828 	if (copy_from_sockptr(&value, optval, sizeof(value)))
829 		return -EFAULT;
830 
831 	if (value > 1)
832 		return -EINVAL;
833 
834 	ctx->zerocopy_sendfile = value;
835 
836 	return 0;
837 }
838 
do_tls_setsockopt_no_pad(struct sock * sk,sockptr_t optval,unsigned int optlen)839 static int do_tls_setsockopt_no_pad(struct sock *sk, sockptr_t optval,
840 				    unsigned int optlen)
841 {
842 	struct tls_context *ctx = tls_get_ctx(sk);
843 	u32 val;
844 	int rc;
845 
846 	if (ctx->prot_info.version != TLS_1_3_VERSION ||
847 	    sockptr_is_null(optval) || optlen < sizeof(val))
848 		return -EINVAL;
849 
850 	rc = copy_from_sockptr(&val, optval, sizeof(val));
851 	if (rc)
852 		return -EFAULT;
853 	if (val > 1)
854 		return -EINVAL;
855 	rc = check_zeroed_sockptr(optval, sizeof(val), optlen - sizeof(val));
856 	if (rc < 1)
857 		return rc == 0 ? -EINVAL : rc;
858 
859 	lock_sock(sk);
860 	rc = -EINVAL;
861 	if (ctx->rx_conf == TLS_SW || ctx->rx_conf == TLS_HW) {
862 		ctx->rx_no_pad = val;
863 		tls_update_rx_zc_capable(ctx);
864 		rc = 0;
865 	}
866 	release_sock(sk);
867 
868 	return rc;
869 }
870 
do_tls_setsockopt(struct sock * sk,int optname,sockptr_t optval,unsigned int optlen)871 static int do_tls_setsockopt(struct sock *sk, int optname, sockptr_t optval,
872 			     unsigned int optlen)
873 {
874 	int rc = 0;
875 
876 	switch (optname) {
877 	case TLS_TX:
878 	case TLS_RX:
879 		lock_sock(sk);
880 		rc = do_tls_setsockopt_conf(sk, optval, optlen,
881 					    optname == TLS_TX);
882 		release_sock(sk);
883 		break;
884 	case TLS_TX_ZEROCOPY_RO:
885 		lock_sock(sk);
886 		rc = do_tls_setsockopt_tx_zc(sk, optval, optlen);
887 		release_sock(sk);
888 		break;
889 	case TLS_RX_EXPECT_NO_PAD:
890 		rc = do_tls_setsockopt_no_pad(sk, optval, optlen);
891 		break;
892 	default:
893 		rc = -ENOPROTOOPT;
894 		break;
895 	}
896 	return rc;
897 }
898 
tls_setsockopt(struct sock * sk,int level,int optname,sockptr_t optval,unsigned int optlen)899 static int tls_setsockopt(struct sock *sk, int level, int optname,
900 			  sockptr_t optval, unsigned int optlen)
901 {
902 	struct tls_context *ctx = tls_get_ctx(sk);
903 
904 	if (level != SOL_TLS)
905 		return ctx->sk_proto->setsockopt(sk, level, optname, optval,
906 						 optlen);
907 
908 	return do_tls_setsockopt(sk, optname, optval, optlen);
909 }
910 
tls_ctx_create(struct sock * sk)911 struct tls_context *tls_ctx_create(struct sock *sk)
912 {
913 	struct inet_connection_sock *icsk = inet_csk(sk);
914 	struct tls_context *ctx;
915 
916 	ctx = kzalloc(sizeof(*ctx), GFP_ATOMIC);
917 	if (!ctx)
918 		return NULL;
919 
920 	mutex_init(&ctx->tx_lock);
921 	rcu_assign_pointer(icsk->icsk_ulp_data, ctx);
922 	ctx->sk_proto = READ_ONCE(sk->sk_prot);
923 	ctx->sk = sk;
924 	return ctx;
925 }
926 
build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG],const struct proto_ops * base)927 static void build_proto_ops(struct proto_ops ops[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
928 			    const struct proto_ops *base)
929 {
930 	ops[TLS_BASE][TLS_BASE] = *base;
931 
932 	ops[TLS_SW  ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
933 	ops[TLS_SW  ][TLS_BASE].sendpage_locked	= tls_sw_sendpage_locked;
934 
935 	ops[TLS_BASE][TLS_SW  ] = ops[TLS_BASE][TLS_BASE];
936 	ops[TLS_BASE][TLS_SW  ].splice_read	= tls_sw_splice_read;
937 
938 	ops[TLS_SW  ][TLS_SW  ] = ops[TLS_SW  ][TLS_BASE];
939 	ops[TLS_SW  ][TLS_SW  ].splice_read	= tls_sw_splice_read;
940 
941 #ifdef CONFIG_TLS_DEVICE
942 	ops[TLS_HW  ][TLS_BASE] = ops[TLS_BASE][TLS_BASE];
943 	ops[TLS_HW  ][TLS_BASE].sendpage_locked	= NULL;
944 
945 	ops[TLS_HW  ][TLS_SW  ] = ops[TLS_BASE][TLS_SW  ];
946 	ops[TLS_HW  ][TLS_SW  ].sendpage_locked	= NULL;
947 
948 	ops[TLS_BASE][TLS_HW  ] = ops[TLS_BASE][TLS_SW  ];
949 
950 	ops[TLS_SW  ][TLS_HW  ] = ops[TLS_SW  ][TLS_SW  ];
951 
952 	ops[TLS_HW  ][TLS_HW  ] = ops[TLS_HW  ][TLS_SW  ];
953 	ops[TLS_HW  ][TLS_HW  ].sendpage_locked	= NULL;
954 #endif
955 #ifdef CONFIG_TLS_TOE
956 	ops[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
957 #endif
958 }
959 
tls_build_proto(struct sock * sk)960 static void tls_build_proto(struct sock *sk)
961 {
962 	int ip_ver = sk->sk_family == AF_INET6 ? TLSV6 : TLSV4;
963 	struct proto *prot = READ_ONCE(sk->sk_prot);
964 
965 	/* Build IPv6 TLS whenever the address of tcpv6 _prot changes */
966 	if (ip_ver == TLSV6 &&
967 	    unlikely(prot != smp_load_acquire(&saved_tcpv6_prot))) {
968 		mutex_lock(&tcpv6_prot_mutex);
969 		if (likely(prot != saved_tcpv6_prot)) {
970 			build_protos(tls_prots[TLSV6], prot);
971 			build_proto_ops(tls_proto_ops[TLSV6],
972 					sk->sk_socket->ops);
973 			smp_store_release(&saved_tcpv6_prot, prot);
974 		}
975 		mutex_unlock(&tcpv6_prot_mutex);
976 	}
977 
978 	if (ip_ver == TLSV4 &&
979 	    unlikely(prot != smp_load_acquire(&saved_tcpv4_prot))) {
980 		mutex_lock(&tcpv4_prot_mutex);
981 		if (likely(prot != saved_tcpv4_prot)) {
982 			build_protos(tls_prots[TLSV4], prot);
983 			build_proto_ops(tls_proto_ops[TLSV4],
984 					sk->sk_socket->ops);
985 			smp_store_release(&saved_tcpv4_prot, prot);
986 		}
987 		mutex_unlock(&tcpv4_prot_mutex);
988 	}
989 }
990 
build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],const struct proto * base)991 static void build_protos(struct proto prot[TLS_NUM_CONFIG][TLS_NUM_CONFIG],
992 			 const struct proto *base)
993 {
994 	prot[TLS_BASE][TLS_BASE] = *base;
995 	prot[TLS_BASE][TLS_BASE].setsockopt	= tls_setsockopt;
996 	prot[TLS_BASE][TLS_BASE].getsockopt	= tls_getsockopt;
997 	prot[TLS_BASE][TLS_BASE].close		= tls_sk_proto_close;
998 
999 	prot[TLS_SW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
1000 	prot[TLS_SW][TLS_BASE].sendmsg		= tls_sw_sendmsg;
1001 	prot[TLS_SW][TLS_BASE].sendpage		= tls_sw_sendpage;
1002 
1003 	prot[TLS_BASE][TLS_SW] = prot[TLS_BASE][TLS_BASE];
1004 	prot[TLS_BASE][TLS_SW].recvmsg		  = tls_sw_recvmsg;
1005 	prot[TLS_BASE][TLS_SW].sock_is_readable   = tls_sw_sock_is_readable;
1006 	prot[TLS_BASE][TLS_SW].close		  = tls_sk_proto_close;
1007 
1008 	prot[TLS_SW][TLS_SW] = prot[TLS_SW][TLS_BASE];
1009 	prot[TLS_SW][TLS_SW].recvmsg		= tls_sw_recvmsg;
1010 	prot[TLS_SW][TLS_SW].sock_is_readable   = tls_sw_sock_is_readable;
1011 	prot[TLS_SW][TLS_SW].close		= tls_sk_proto_close;
1012 
1013 #ifdef CONFIG_TLS_DEVICE
1014 	prot[TLS_HW][TLS_BASE] = prot[TLS_BASE][TLS_BASE];
1015 	prot[TLS_HW][TLS_BASE].sendmsg		= tls_device_sendmsg;
1016 	prot[TLS_HW][TLS_BASE].sendpage		= tls_device_sendpage;
1017 
1018 	prot[TLS_HW][TLS_SW] = prot[TLS_BASE][TLS_SW];
1019 	prot[TLS_HW][TLS_SW].sendmsg		= tls_device_sendmsg;
1020 	prot[TLS_HW][TLS_SW].sendpage		= tls_device_sendpage;
1021 
1022 	prot[TLS_BASE][TLS_HW] = prot[TLS_BASE][TLS_SW];
1023 
1024 	prot[TLS_SW][TLS_HW] = prot[TLS_SW][TLS_SW];
1025 
1026 	prot[TLS_HW][TLS_HW] = prot[TLS_HW][TLS_SW];
1027 #endif
1028 #ifdef CONFIG_TLS_TOE
1029 	prot[TLS_HW_RECORD][TLS_HW_RECORD] = *base;
1030 	prot[TLS_HW_RECORD][TLS_HW_RECORD].hash		= tls_toe_hash;
1031 	prot[TLS_HW_RECORD][TLS_HW_RECORD].unhash	= tls_toe_unhash;
1032 #endif
1033 }
1034 
tls_init(struct sock * sk)1035 static int tls_init(struct sock *sk)
1036 {
1037 	struct tls_context *ctx;
1038 	int rc = 0;
1039 
1040 	tls_build_proto(sk);
1041 
1042 #ifdef CONFIG_TLS_TOE
1043 	if (tls_toe_bypass(sk))
1044 		return 0;
1045 #endif
1046 
1047 	/* The TLS ulp is currently supported only for TCP sockets
1048 	 * in ESTABLISHED state.
1049 	 * Supporting sockets in LISTEN state will require us
1050 	 * to modify the accept implementation to clone rather then
1051 	 * share the ulp context.
1052 	 */
1053 	if (sk->sk_state != TCP_ESTABLISHED)
1054 		return -ENOTCONN;
1055 
1056 	/* allocate tls context */
1057 	write_lock_bh(&sk->sk_callback_lock);
1058 	ctx = tls_ctx_create(sk);
1059 	if (!ctx) {
1060 		rc = -ENOMEM;
1061 		goto out;
1062 	}
1063 
1064 	ctx->tx_conf = TLS_BASE;
1065 	ctx->rx_conf = TLS_BASE;
1066 	update_sk_prot(sk, ctx);
1067 out:
1068 	write_unlock_bh(&sk->sk_callback_lock);
1069 	return rc;
1070 }
1071 
tls_update(struct sock * sk,struct proto * p,void (* write_space)(struct sock * sk))1072 static void tls_update(struct sock *sk, struct proto *p,
1073 		       void (*write_space)(struct sock *sk))
1074 {
1075 	struct tls_context *ctx;
1076 
1077 	WARN_ON_ONCE(sk->sk_prot == p);
1078 
1079 	ctx = tls_get_ctx(sk);
1080 	if (likely(ctx)) {
1081 		ctx->sk_write_space = write_space;
1082 		ctx->sk_proto = p;
1083 	} else {
1084 		/* Pairs with lockless read in sk_clone_lock(). */
1085 		WRITE_ONCE(sk->sk_prot, p);
1086 		sk->sk_write_space = write_space;
1087 	}
1088 }
1089 
tls_user_config(struct tls_context * ctx,bool tx)1090 static u16 tls_user_config(struct tls_context *ctx, bool tx)
1091 {
1092 	u16 config = tx ? ctx->tx_conf : ctx->rx_conf;
1093 
1094 	switch (config) {
1095 	case TLS_BASE:
1096 		return TLS_CONF_BASE;
1097 	case TLS_SW:
1098 		return TLS_CONF_SW;
1099 	case TLS_HW:
1100 		return TLS_CONF_HW;
1101 	case TLS_HW_RECORD:
1102 		return TLS_CONF_HW_RECORD;
1103 	}
1104 	return 0;
1105 }
1106 
tls_get_info(const struct sock * sk,struct sk_buff * skb)1107 static int tls_get_info(const struct sock *sk, struct sk_buff *skb)
1108 {
1109 	u16 version, cipher_type;
1110 	struct tls_context *ctx;
1111 	struct nlattr *start;
1112 	int err;
1113 
1114 	start = nla_nest_start_noflag(skb, INET_ULP_INFO_TLS);
1115 	if (!start)
1116 		return -EMSGSIZE;
1117 
1118 	rcu_read_lock();
1119 	ctx = rcu_dereference(inet_csk(sk)->icsk_ulp_data);
1120 	if (!ctx) {
1121 		err = 0;
1122 		goto nla_failure;
1123 	}
1124 	version = ctx->prot_info.version;
1125 	if (version) {
1126 		err = nla_put_u16(skb, TLS_INFO_VERSION, version);
1127 		if (err)
1128 			goto nla_failure;
1129 	}
1130 	cipher_type = ctx->prot_info.cipher_type;
1131 	if (cipher_type) {
1132 		err = nla_put_u16(skb, TLS_INFO_CIPHER, cipher_type);
1133 		if (err)
1134 			goto nla_failure;
1135 	}
1136 	err = nla_put_u16(skb, TLS_INFO_TXCONF, tls_user_config(ctx, true));
1137 	if (err)
1138 		goto nla_failure;
1139 
1140 	err = nla_put_u16(skb, TLS_INFO_RXCONF, tls_user_config(ctx, false));
1141 	if (err)
1142 		goto nla_failure;
1143 
1144 	if (ctx->tx_conf == TLS_HW && ctx->zerocopy_sendfile) {
1145 		err = nla_put_flag(skb, TLS_INFO_ZC_RO_TX);
1146 		if (err)
1147 			goto nla_failure;
1148 	}
1149 	if (ctx->rx_no_pad) {
1150 		err = nla_put_flag(skb, TLS_INFO_RX_NO_PAD);
1151 		if (err)
1152 			goto nla_failure;
1153 	}
1154 
1155 	rcu_read_unlock();
1156 	nla_nest_end(skb, start);
1157 	return 0;
1158 
1159 nla_failure:
1160 	rcu_read_unlock();
1161 	nla_nest_cancel(skb, start);
1162 	return err;
1163 }
1164 
tls_get_info_size(const struct sock * sk)1165 static size_t tls_get_info_size(const struct sock *sk)
1166 {
1167 	size_t size = 0;
1168 
1169 	size += nla_total_size(0) +		/* INET_ULP_INFO_TLS */
1170 		nla_total_size(sizeof(u16)) +	/* TLS_INFO_VERSION */
1171 		nla_total_size(sizeof(u16)) +	/* TLS_INFO_CIPHER */
1172 		nla_total_size(sizeof(u16)) +	/* TLS_INFO_RXCONF */
1173 		nla_total_size(sizeof(u16)) +	/* TLS_INFO_TXCONF */
1174 		nla_total_size(0) +		/* TLS_INFO_ZC_RO_TX */
1175 		nla_total_size(0) +		/* TLS_INFO_RX_NO_PAD */
1176 		0;
1177 
1178 	return size;
1179 }
1180 
tls_init_net(struct net * net)1181 static int __net_init tls_init_net(struct net *net)
1182 {
1183 	int err;
1184 
1185 	net->mib.tls_statistics = alloc_percpu(struct linux_tls_mib);
1186 	if (!net->mib.tls_statistics)
1187 		return -ENOMEM;
1188 
1189 	err = tls_proc_init(net);
1190 	if (err)
1191 		goto err_free_stats;
1192 
1193 	return 0;
1194 err_free_stats:
1195 	free_percpu(net->mib.tls_statistics);
1196 	return err;
1197 }
1198 
tls_exit_net(struct net * net)1199 static void __net_exit tls_exit_net(struct net *net)
1200 {
1201 	tls_proc_fini(net);
1202 	free_percpu(net->mib.tls_statistics);
1203 }
1204 
1205 static struct pernet_operations tls_proc_ops = {
1206 	.init = tls_init_net,
1207 	.exit = tls_exit_net,
1208 };
1209 
1210 static struct tcp_ulp_ops tcp_tls_ulp_ops __read_mostly = {
1211 	.name			= "tls",
1212 	.owner			= THIS_MODULE,
1213 	.init			= tls_init,
1214 	.update			= tls_update,
1215 	.get_info		= tls_get_info,
1216 	.get_info_size		= tls_get_info_size,
1217 };
1218 
tls_register(void)1219 static int __init tls_register(void)
1220 {
1221 	int err;
1222 
1223 	err = register_pernet_subsys(&tls_proc_ops);
1224 	if (err)
1225 		return err;
1226 
1227 	err = tls_strp_dev_init();
1228 	if (err)
1229 		goto err_pernet;
1230 
1231 	err = tls_device_init();
1232 	if (err)
1233 		goto err_strp;
1234 
1235 	tcp_register_ulp(&tcp_tls_ulp_ops);
1236 
1237 	return 0;
1238 err_strp:
1239 	tls_strp_dev_exit();
1240 err_pernet:
1241 	unregister_pernet_subsys(&tls_proc_ops);
1242 	return err;
1243 }
1244 
tls_unregister(void)1245 static void __exit tls_unregister(void)
1246 {
1247 	tcp_unregister_ulp(&tcp_tls_ulp_ops);
1248 	tls_strp_dev_exit();
1249 	tls_device_cleanup();
1250 	unregister_pernet_subsys(&tls_proc_ops);
1251 }
1252 
1253 module_init(tls_register);
1254 module_exit(tls_unregister);
1255