43 void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,  in rds_inc_init()  argument
46 	refcount_set(&inc->i_refcount, 1);  in rds_inc_init()
47 	INIT_LIST_HEAD(&inc->i_item);  in rds_inc_init()
48 	inc->i_conn = conn;  in rds_inc_init()
49 	inc->i_saddr = *saddr;  in rds_inc_init()
50 	inc->i_usercopy.rdma_cookie = 0;  in rds_inc_init()
51 	inc->i_usercopy.rx_tstamp = ktime_set(0, 0);  in rds_inc_init()
53 	memset(inc->i_rx_lat_trace, 0, sizeof(inc->i_rx_lat_trace));  in rds_inc_init()
57 void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *cp,  in rds_inc_path_init()  argument
60 	refcount_set(&inc->i_refcount, 1);  in rds_inc_path_init()
61 	INIT_LIST_HEAD(&inc->i_item);  in rds_inc_path_init()
62 	inc->i_conn = cp->cp_conn;  in rds_inc_path_init()
63 	inc->i_conn_path = cp;  in rds_inc_path_init()
64 	inc->i_saddr = *saddr;  in rds_inc_path_init()
65 	inc->i_usercopy.rdma_cookie = 0;  in rds_inc_path_init()
66 	inc->i_usercopy.rx_tstamp = ktime_set(0, 0);  in rds_inc_path_init()
70 static void rds_inc_addref(struct rds_incoming *inc)  in rds_inc_addref()  argument
72 	rdsdebug("addref inc %p ref %d\n", inc, refcount_read(&inc->i_refcount));  in rds_inc_addref()
73 	refcount_inc(&inc->i_refcount);  in rds_inc_addref()
76 void rds_inc_put(struct rds_incoming *inc)  in rds_inc_put()  argument
78 	rdsdebug("put inc %p ref %d\n", inc, refcount_read(&inc->i_refcount));  in rds_inc_put()
79 	if (refcount_dec_and_test(&inc->i_refcount)) {  in rds_inc_put()
80 		BUG_ON(!list_empty(&inc->i_item));  in rds_inc_put()
82 		inc->i_conn->c_trans->inc_free(inc);  in rds_inc_put()
165 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs)  in rds_recv_incoming_exthdrs()  argument
167 	struct rds_header *hdr = &inc->i_hdr;  in rds_recv_incoming_exthdrs()
189 			inc->i_usercopy.rdma_cookie = rds_rdma_make_cookie(  in rds_recv_incoming_exthdrs()
284 		       struct rds_incoming *inc, gfp_t gfp)  in rds_recv_incoming()  argument
291 	inc->i_conn = conn;  in rds_recv_incoming()
292 	inc->i_rx_jiffies = jiffies;  in rds_recv_incoming()
294 		cp = inc->i_conn_path;  in rds_recv_incoming()
301 		 inc,  in rds_recv_incoming()
302 		 (unsigned long long)be64_to_cpu(inc->i_hdr.h_sequence),  in rds_recv_incoming()
303 		 be32_to_cpu(inc->i_hdr.h_len),  in rds_recv_incoming()
304 		 be16_to_cpu(inc->i_hdr.h_sport),  in rds_recv_incoming()
305 		 be16_to_cpu(inc->i_hdr.h_dport),  in rds_recv_incoming()
306 		 inc->i_hdr.h_flags,  in rds_recv_incoming()
307 		 inc->i_rx_jiffies);  in rds_recv_incoming()
329 	if (be64_to_cpu(inc->i_hdr.h_sequence) < cp->cp_next_rx_seq &&  in rds_recv_incoming()
330 	    (inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) {  in rds_recv_incoming()
334 	cp->cp_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1;  in rds_recv_incoming()
336 	if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) {  in rds_recv_incoming()
337 		if (inc->i_hdr.h_sport == 0) {  in rds_recv_incoming()
343 		rds_send_pong(cp, inc->i_hdr.h_sport);  in rds_recv_incoming()
345 		if (RDS_HS_PROBE(be16_to_cpu(inc->i_hdr.h_sport),  in rds_recv_incoming()
346 				 be16_to_cpu(inc->i_hdr.h_dport))) {  in rds_recv_incoming()
347 			rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn);  in rds_recv_incoming()
353 	if (be16_to_cpu(inc->i_hdr.h_dport) ==  RDS_FLAG_PROBE_PORT &&  in rds_recv_incoming()
354 	    inc->i_hdr.h_sport == 0) {  in rds_recv_incoming()
355 		rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn);  in rds_recv_incoming()
362 	rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if);  in rds_recv_incoming()
369 	rds_recv_incoming_exthdrs(inc, rs);  in rds_recv_incoming()
377 		rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);  in rds_recv_incoming()
379 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_recv_incoming()
380 				      be32_to_cpu(inc->i_hdr.h_len),  in rds_recv_incoming()
381 				      inc->i_hdr.h_dport);  in rds_recv_incoming()
383 			inc->i_usercopy.rx_tstamp = ktime_get_real();  in rds_recv_incoming()
384 		rds_inc_addref(inc);  in rds_recv_incoming()
385 		inc->i_rx_lat_trace[RDS_MSG_RX_END] = local_clock();  in rds_recv_incoming()
386 		list_add_tail(&inc->i_item, &rs->rs_recv_queue);  in rds_recv_incoming()
403 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc)  in rds_next_incoming()  argument
407 	if (!*inc) {  in rds_next_incoming()
410 			*inc = list_entry(rs->rs_recv_queue.next,  in rds_next_incoming()
413 			rds_inc_addref(*inc);  in rds_next_incoming()
418 	return *inc != NULL;  in rds_next_incoming()
421 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc,  in rds_still_queued()  argument
429 	if (!list_empty(&inc->i_item)) {  in rds_still_queued()
433 			rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_still_queued()
434 					      -be32_to_cpu(inc->i_hdr.h_len),  in rds_still_queued()
435 					      inc->i_hdr.h_dport);  in rds_still_queued()
436 			list_del_init(&inc->i_item);  in rds_still_queued()
437 			rds_inc_put(inc);  in rds_still_queued()
442 	rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop);  in rds_still_queued()
539 static int rds_cmsg_recv(struct rds_incoming *inc, struct msghdr *msg,  in rds_cmsg_recv()  argument
544 	if (inc->i_usercopy.rdma_cookie) {  in rds_cmsg_recv()
546 				sizeof(inc->i_usercopy.rdma_cookie),  in rds_cmsg_recv()
547 				&inc->i_usercopy.rdma_cookie);  in rds_cmsg_recv()
552 	if ((inc->i_usercopy.rx_tstamp != 0) &&  in rds_cmsg_recv()
555 			ns_to_kernel_old_timeval(inc->i_usercopy.rx_tstamp);  in rds_cmsg_recv()
579 		inc->i_rx_lat_trace[RDS_MSG_RX_CMSG] = local_clock();  in rds_cmsg_recv()
584 			t.rx_trace[i] = inc->i_rx_lat_trace[j + 1] -  in rds_cmsg_recv()
585 					  inc->i_rx_lat_trace[j];  in rds_cmsg_recv()
642 	struct rds_incoming *inc = NULL;  in rds_recvmsg()  local
666 		if (!rds_next_incoming(rs, &inc)) {  in rds_recvmsg()
677 					 rds_next_incoming(rs, &inc)), timeo);  in rds_recvmsg()
678 			rdsdebug("recvmsg woke inc %p timeo %ld\n", inc,  in rds_recvmsg()
689 		rdsdebug("copying inc %p from %pI6c:%u to user\n", inc,  in rds_recvmsg()
690 			 &inc->i_conn->c_faddr,  in rds_recvmsg()
691 			 ntohs(inc->i_hdr.h_sport));  in rds_recvmsg()
692 		ret = inc->i_conn->c_trans->inc_copy_to_user(inc, &msg->msg_iter);  in rds_recvmsg()
701 		if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {  in rds_recvmsg()
702 			rds_inc_put(inc);  in rds_recvmsg()
703 			inc = NULL;  in rds_recvmsg()
709 		if (ret < be32_to_cpu(inc->i_hdr.h_len)) {  in rds_recvmsg()
711 				ret = be32_to_cpu(inc->i_hdr.h_len);  in rds_recvmsg()
715 		if (rds_cmsg_recv(inc, msg, rs)) {  in rds_recvmsg()
724 			if (ipv6_addr_v4mapped(&inc->i_saddr)) {  in rds_recvmsg()
726 				sin->sin_port = inc->i_hdr.h_sport;  in rds_recvmsg()
728 				    inc->i_saddr.s6_addr32[3];  in rds_recvmsg()
733 				sin6->sin6_port = inc->i_hdr.h_sport;  in rds_recvmsg()
734 				sin6->sin6_addr = inc->i_saddr;  in rds_recvmsg()
743 	if (inc)  in rds_recvmsg()
744 		rds_inc_put(inc);  in rds_recvmsg()
758 	struct rds_incoming *inc, *tmp;  in rds_clear_recv_queue()  local
762 	list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) {  in rds_clear_recv_queue()
763 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_clear_recv_queue()
764 				      -be32_to_cpu(inc->i_hdr.h_len),  in rds_clear_recv_queue()
765 				      inc->i_hdr.h_dport);  in rds_clear_recv_queue()
766 		list_del_init(&inc->i_item);  in rds_clear_recv_queue()
767 		rds_inc_put(inc);  in rds_clear_recv_queue()
776 void rds_inc_info_copy(struct rds_incoming *inc,  in rds_inc_info_copy()  argument
782 	minfo.seq = be64_to_cpu(inc->i_hdr.h_sequence);  in rds_inc_info_copy()
783 	minfo.len = be32_to_cpu(inc->i_hdr.h_len);  in rds_inc_info_copy()
784 	minfo.tos = inc->i_conn->c_tos;  in rds_inc_info_copy()
789 		minfo.lport = inc->i_hdr.h_dport;  in rds_inc_info_copy()
790 		minfo.fport = inc->i_hdr.h_sport;  in rds_inc_info_copy()
794 		minfo.lport = inc->i_hdr.h_sport;  in rds_inc_info_copy()
795 		minfo.fport = inc->i_hdr.h_dport;  in rds_inc_info_copy()
804 void rds6_inc_info_copy(struct rds_incoming *inc,  in rds6_inc_info_copy()  argument
811 	minfo6.seq = be64_to_cpu(inc->i_hdr.h_sequence);  in rds6_inc_info_copy()
812 	minfo6.len = be32_to_cpu(inc->i_hdr.h_len);  in rds6_inc_info_copy()
813 	minfo6.tos = inc->i_conn->c_tos;  in rds6_inc_info_copy()
818 		minfo6.lport = inc->i_hdr.h_dport;  in rds6_inc_info_copy()
819 		minfo6.fport = inc->i_hdr.h_sport;  in rds6_inc_info_copy()
823 		minfo6.lport = inc->i_hdr.h_sport;  in rds6_inc_info_copy()
824 		minfo6.fport = inc->i_hdr.h_dport;  in rds6_inc_info_copy()