32 static int bpf_tcp_ingress(struct sock *sk, struct sk_psock *psock,  in bpf_tcp_ingress()  argument
77 		sk_psock_queue_msg(psock, tmp);  in bpf_tcp_ingress()
78 		sk_psock_data_ready(sk, psock);  in bpf_tcp_ingress()
164 	struct sk_psock *psock = sk_psock_get(sk);  in tcp_bpf_sendmsg_redir()  local
167 	if (unlikely(!psock))  in tcp_bpf_sendmsg_redir()
170 	ret = ingress ? bpf_tcp_ingress(sk, psock, msg, bytes, flags) :  in tcp_bpf_sendmsg_redir()
172 	sk_psock_put(sk, psock);  in tcp_bpf_sendmsg_redir()
178 static int tcp_msg_wait_data(struct sock *sk, struct sk_psock *psock,  in tcp_msg_wait_data()  argument
193 			    !list_empty(&psock->ingress_msg) ||  in tcp_msg_wait_data()
200 static bool is_next_msg_fin(struct sk_psock *psock)  in is_next_msg_fin()  argument
206 	msg_rx = sk_psock_peek_msg(psock);  in is_next_msg_fin()
227 	struct sk_psock *psock;  in tcp_bpf_recvmsg_parser()  local
236 	psock = sk_psock_get(sk);  in tcp_bpf_recvmsg_parser()
237 	if (unlikely(!psock))  in tcp_bpf_recvmsg_parser()
262 	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg_parser()
269 		bool is_fin = is_next_msg_fin(psock);  in tcp_bpf_recvmsg_parser()
309 		data = tcp_msg_wait_data(sk, psock, timeo);  in tcp_bpf_recvmsg_parser()
314 		if (data && !sk_psock_queue_empty(psock))  in tcp_bpf_recvmsg_parser()
327 	sk_psock_put(sk, psock);  in tcp_bpf_recvmsg_parser()
334 	struct sk_psock *psock;  in tcp_bpf_recvmsg()  local
343 	psock = sk_psock_get(sk);  in tcp_bpf_recvmsg()
344 	if (unlikely(!psock))  in tcp_bpf_recvmsg()
347 	    sk_psock_queue_empty(psock)) {  in tcp_bpf_recvmsg()
348 		sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
353 	copied = sk_msg_recvmsg(sk, psock, msg, len, flags);  in tcp_bpf_recvmsg()
359 		data = tcp_msg_wait_data(sk, psock, timeo);  in tcp_bpf_recvmsg()
365 			if (!sk_psock_queue_empty(psock))  in tcp_bpf_recvmsg()
368 			sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
377 	sk_psock_put(sk, psock);  in tcp_bpf_recvmsg()
381 static int tcp_bpf_send_verdict(struct sock *sk, struct sk_psock *psock,  in tcp_bpf_send_verdict()  argument
391 	if (psock->eval == __SK_NONE) {  in tcp_bpf_send_verdict()
398 		psock->eval = sk_psock_msg_verdict(sk, psock, msg);  in tcp_bpf_send_verdict()
404 		psock->cork_bytes = msg->cork_bytes - msg->sg.size;  in tcp_bpf_send_verdict()
405 		if (!psock->cork) {  in tcp_bpf_send_verdict()
406 			psock->cork = kzalloc(sizeof(*psock->cork),  in tcp_bpf_send_verdict()
408 			if (!psock->cork)  in tcp_bpf_send_verdict()
411 		memcpy(psock->cork, msg, sizeof(*msg));  in tcp_bpf_send_verdict()
416 	if (psock->apply_bytes && psock->apply_bytes < tosend)  in tcp_bpf_send_verdict()
417 		tosend = psock->apply_bytes;  in tcp_bpf_send_verdict()
420 	switch (psock->eval) {  in tcp_bpf_send_verdict()
427 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
430 		redir_ingress = psock->redir_ingress;  in tcp_bpf_send_verdict()
431 		sk_redir = psock->sk_redir;  in tcp_bpf_send_verdict()
432 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
433 		if (!psock->apply_bytes) {  in tcp_bpf_send_verdict()
435 			eval = psock->eval;  in tcp_bpf_send_verdict()
436 			psock->eval = __SK_NONE;  in tcp_bpf_send_verdict()
437 			psock->sk_redir = NULL;  in tcp_bpf_send_verdict()
439 		if (psock->cork) {  in tcp_bpf_send_verdict()
441 			psock->cork = NULL;  in tcp_bpf_send_verdict()
471 		sk_msg_apply_bytes(psock, tosend);  in tcp_bpf_send_verdict()
477 		if (!psock->apply_bytes) {  in tcp_bpf_send_verdict()
478 			psock->eval =  __SK_NONE;  in tcp_bpf_send_verdict()
479 			if (psock->sk_redir) {  in tcp_bpf_send_verdict()
480 				sock_put(psock->sk_redir);  in tcp_bpf_send_verdict()
481 				psock->sk_redir = NULL;  in tcp_bpf_send_verdict()
499 	struct sk_psock *psock;  in tcp_bpf_sendmsg()  local
507 	psock = sk_psock_get(sk);  in tcp_bpf_sendmsg()
508 	if (unlikely(!psock))  in tcp_bpf_sendmsg()
525 		if (psock->cork) {  in tcp_bpf_sendmsg()
526 			msg_tx = psock->cork;  in tcp_bpf_sendmsg()
549 		if (psock->cork_bytes) {  in tcp_bpf_sendmsg()
550 			if (size > psock->cork_bytes)  in tcp_bpf_sendmsg()
551 				psock->cork_bytes = 0;  in tcp_bpf_sendmsg()
553 				psock->cork_bytes -= size;  in tcp_bpf_sendmsg()
554 			if (psock->cork_bytes && !enospc)  in tcp_bpf_sendmsg()
557 			psock->eval = __SK_NONE;  in tcp_bpf_sendmsg()
558 			psock->cork_bytes = 0;  in tcp_bpf_sendmsg()
561 		err = tcp_bpf_send_verdict(sk, psock, msg_tx, &copied, flags);  in tcp_bpf_sendmsg()
570 			if (msg_tx && msg_tx != psock->cork)  in tcp_bpf_sendmsg()
579 	sk_psock_put(sk, psock);  in tcp_bpf_sendmsg()
649 int tcp_bpf_update_proto(struct sock *sk, struct sk_psock *psock, bool restore)  in tcp_bpf_update_proto()  argument
652 	int config = psock->progs.msg_parser   ? TCP_BPF_TX   : TCP_BPF_BASE;  in tcp_bpf_update_proto()
654 	if (psock->progs.stream_verdict || psock->progs.skb_verdict) {  in tcp_bpf_update_proto()
665 			WRITE_ONCE(sk->sk_prot->unhash, psock->saved_unhash);  in tcp_bpf_update_proto()
666 			tcp_update_ulp(sk, psock->sk_proto, psock->saved_write_space);  in tcp_bpf_update_proto()
668 			sk->sk_write_space = psock->saved_write_space;  in tcp_bpf_update_proto()
670 			sock_replace_proto(sk, psock->sk_proto);  in tcp_bpf_update_proto()
676 		if (tcp_bpf_assert_proto_ops(psock->sk_proto))  in tcp_bpf_update_proto()
679 		tcp_bpf_check_v6_needs_rebuild(psock->sk_proto);  in tcp_bpf_update_proto()