88 static void rds_recv_rcvbuf_delta(struct rds_sock *rs, struct sock *sk,  in rds_recv_rcvbuf_delta()  argument
97 	rs->rs_rcv_bytes += delta;  in rds_recv_rcvbuf_delta()
104 	if (rs->rs_transport->t_type == RDS_TRANS_LOOP)  in rds_recv_rcvbuf_delta()
107 	now_congested = rs->rs_rcv_bytes > rds_sk_rcvbuf(rs);  in rds_recv_rcvbuf_delta()
111 	  rs, &rs->rs_bound_addr,  in rds_recv_rcvbuf_delta()
112 	  ntohs(rs->rs_bound_port), rs->rs_rcv_bytes,  in rds_recv_rcvbuf_delta()
113 	  rds_sk_rcvbuf(rs), now_congested, delta);  in rds_recv_rcvbuf_delta()
116 	if (!rs->rs_congested && now_congested) {  in rds_recv_rcvbuf_delta()
117 		rs->rs_congested = 1;  in rds_recv_rcvbuf_delta()
124 	else if (rs->rs_congested && (rs->rs_rcv_bytes < (rds_sk_rcvbuf(rs)/2))) {  in rds_recv_rcvbuf_delta()
125 		rs->rs_congested = 0;  in rds_recv_rcvbuf_delta()
166 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs)  in rds_recv_incoming_exthdrs()  argument
184 			rds_rdma_unuse(rs, be32_to_cpu(buffer.rdma.h_rdma_rkey), 0);  in rds_recv_incoming_exthdrs()
287 	struct rds_sock *rs = NULL;  in rds_recv_incoming()  local
363 	rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if);  in rds_recv_incoming()
364 	if (!rs) {  in rds_recv_incoming()
370 	rds_recv_incoming_exthdrs(inc, rs);  in rds_recv_incoming()
373 	sk = rds_rs_to_sk(rs);  in rds_recv_incoming()
376 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
378 		rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs);  in rds_recv_incoming()
380 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_recv_incoming()
387 		list_add_tail(&inc->i_item, &rs->rs_recv_queue);  in rds_recv_incoming()
392 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_recv_incoming()
395 	if (rs)  in rds_recv_incoming()
396 		rds_sock_put(rs);  in rds_recv_incoming()
404 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc)  in rds_next_incoming()  argument
409 		read_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_next_incoming()
410 		if (!list_empty(&rs->rs_recv_queue)) {  in rds_next_incoming()
411 			*inc = list_entry(rs->rs_recv_queue.next,  in rds_next_incoming()
416 		read_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_next_incoming()
422 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc,  in rds_still_queued()  argument
425 	struct sock *sk = rds_rs_to_sk(rs);  in rds_still_queued()
429 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_still_queued()
434 			rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_still_queued()
441 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_still_queued()
443 	rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop);  in rds_still_queued()
451 int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_queue_get()  argument
475 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
476 	while (!list_empty(&rs->rs_notify_queue) && count < max_messages) {  in rds_notify_queue_get()
477 		notifier = list_entry(rs->rs_notify_queue.next,  in rds_notify_queue_get()
482 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
508 		spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_queue_get()
509 		list_splice(&copy, &rs->rs_notify_queue);  in rds_notify_queue_get()
510 		spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_queue_get()
519 static int rds_notify_cong(struct rds_sock *rs, struct msghdr *msghdr)  in rds_notify_cong()  argument
521 	uint64_t notify = rs->rs_cong_notify;  in rds_notify_cong()
530 	spin_lock_irqsave(&rs->rs_lock, flags);  in rds_notify_cong()
531 	rs->rs_cong_notify &= ~notify;  in rds_notify_cong()
532 	spin_unlock_irqrestore(&rs->rs_lock, flags);  in rds_notify_cong()
541 			 struct rds_sock *rs)  in rds_cmsg_recv()  argument
554 	    sock_flag(rds_rs_to_sk(rs), SOCK_RCVTSTAMP)) {  in rds_cmsg_recv()
558 		if (!sock_flag(rds_rs_to_sk(rs), SOCK_TSTAMP_NEW)) {  in rds_cmsg_recv()
575 	if (rs->rs_rx_traces) {  in rds_cmsg_recv()
581 		t.rx_traces =  rs->rs_rx_traces;  in rds_cmsg_recv()
582 		for (i = 0; i < rs->rs_rx_traces; i++) {  in rds_cmsg_recv()
583 			j = rs->rs_rx_trace[i];  in rds_cmsg_recv()
599 static bool rds_recvmsg_zcookie(struct rds_sock *rs, struct msghdr *msg)  in rds_recvmsg_zcookie()  argument
601 	struct rds_msg_zcopy_queue *q = &rs->rs_zcookie_queue;  in rds_recvmsg_zcookie()
609 	if (!sock_flag(rds_rs_to_sk(rs), SOCK_ZEROCOPY) ||  in rds_recvmsg_zcookie()
638 	struct rds_sock *rs = rds_sk_to_rs(sk);  in rds_recvmsg()  local
657 		if (!list_empty(&rs->rs_notify_queue)) {  in rds_recvmsg()
658 			ret = rds_notify_queue_get(rs, msg);  in rds_recvmsg()
662 		if (rs->rs_cong_notify) {  in rds_recvmsg()
663 			ret = rds_notify_cong(rs, msg);  in rds_recvmsg()
667 		if (!rds_next_incoming(rs, &inc)) {  in rds_recvmsg()
669 				bool reaped = rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
676 					(!list_empty(&rs->rs_notify_queue) ||  in rds_recvmsg()
677 					 rs->rs_cong_notify ||  in rds_recvmsg()
678 					 rds_next_incoming(rs, &inc)), timeo);  in rds_recvmsg()
702 		if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) {  in rds_recvmsg()
716 		if (rds_cmsg_recv(inc, msg, rs)) {  in rds_recvmsg()
720 		rds_recvmsg_zcookie(rs, msg);  in rds_recvmsg()
737 				sin6->sin6_scope_id = rs->rs_bound_scope_id;  in rds_recvmsg()
756 void rds_clear_recv_queue(struct rds_sock *rs)  in rds_clear_recv_queue()  argument
758 	struct sock *sk = rds_rs_to_sk(rs);  in rds_clear_recv_queue()
762 	write_lock_irqsave(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()
763 	list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) {  in rds_clear_recv_queue()
764 		rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong,  in rds_clear_recv_queue()
770 	write_unlock_irqrestore(&rs->rs_recv_lock, flags);  in rds_clear_recv_queue()