61 void rds_send_reset(struct rds_connection *conn)  in rds_send_reset()  argument
66 	if (conn->c_xmit_rm) {  in rds_send_reset()
67 		rm = conn->c_xmit_rm;  in rds_send_reset()
68 		conn->c_xmit_rm = NULL;  in rds_send_reset()
77 	conn->c_xmit_sg = 0;  in rds_send_reset()
78 	conn->c_xmit_hdr_off = 0;  in rds_send_reset()
79 	conn->c_xmit_data_off = 0;  in rds_send_reset()
80 	conn->c_xmit_atomic_sent = 0;  in rds_send_reset()
81 	conn->c_xmit_rdma_sent = 0;  in rds_send_reset()
82 	conn->c_xmit_data_sent = 0;  in rds_send_reset()
84 	conn->c_map_queued = 0;  in rds_send_reset()
86 	conn->c_unacked_packets = rds_sysctl_max_unacked_packets;  in rds_send_reset()
87 	conn->c_unacked_bytes = rds_sysctl_max_unacked_bytes;  in rds_send_reset()
90 	spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_reset()
91 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_reset()
95 	list_splice_init(&conn->c_retrans, &conn->c_send_queue);  in rds_send_reset()
96 	spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_reset()
99 static int acquire_in_xmit(struct rds_connection *conn)  in acquire_in_xmit()  argument
101 	return test_and_set_bit(RDS_IN_XMIT, &conn->c_flags) == 0;  in acquire_in_xmit()
104 static void release_in_xmit(struct rds_connection *conn)  in release_in_xmit()  argument
106 	clear_bit(RDS_IN_XMIT, &conn->c_flags);  in release_in_xmit()
114 	if (waitqueue_active(&conn->c_waitq))  in release_in_xmit()
115 		wake_up_all(&conn->c_waitq);  in release_in_xmit()
132 int rds_send_xmit(struct rds_connection *conn)  in rds_send_xmit()  argument
150 	if (!acquire_in_xmit(conn)) {  in rds_send_xmit()
160 	if (!rds_conn_up(conn)) {  in rds_send_xmit()
161 		release_in_xmit(conn);  in rds_send_xmit()
166 	if (conn->c_trans->xmit_prepare)  in rds_send_xmit()
167 		conn->c_trans->xmit_prepare(conn);  in rds_send_xmit()
175 		rm = conn->c_xmit_rm;  in rds_send_xmit()
181 		if (!rm && test_and_clear_bit(0, &conn->c_map_queued)) {  in rds_send_xmit()
182 			rm = rds_cong_update_alloc(conn);  in rds_send_xmit()
189 			conn->c_xmit_rm = rm;  in rds_send_xmit()
202 			spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_xmit()
204 			if (!list_empty(&conn->c_send_queue)) {  in rds_send_xmit()
205 				rm = list_entry(conn->c_send_queue.next,  in rds_send_xmit()
214 				list_move_tail(&rm->m_conn_item, &conn->c_retrans);  in rds_send_xmit()
217 			spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_xmit()
231 				spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_xmit()
234 				spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_xmit()
240 			if (conn->c_unacked_packets == 0 ||  in rds_send_xmit()
241 			    conn->c_unacked_bytes < len) {  in rds_send_xmit()
244 				conn->c_unacked_packets = rds_sysctl_max_unacked_packets;  in rds_send_xmit()
245 				conn->c_unacked_bytes = rds_sysctl_max_unacked_bytes;  in rds_send_xmit()
248 				conn->c_unacked_bytes -= len;  in rds_send_xmit()
249 				conn->c_unacked_packets--;  in rds_send_xmit()
252 			conn->c_xmit_rm = rm;  in rds_send_xmit()
256 		if (rm->rdma.op_active && !conn->c_xmit_rdma_sent) {  in rds_send_xmit()
258 			ret = conn->c_trans->xmit_rdma(conn, &rm->rdma);  in rds_send_xmit()
261 			conn->c_xmit_rdma_sent = 1;  in rds_send_xmit()
268 		if (rm->atomic.op_active && !conn->c_xmit_atomic_sent) {  in rds_send_xmit()
270 			ret = conn->c_trans->xmit_atomic(conn, &rm->atomic);  in rds_send_xmit()
273 			conn->c_xmit_atomic_sent = 1;  in rds_send_xmit()
302 		if (rm->data.op_active && !conn->c_xmit_data_sent) {  in rds_send_xmit()
304 			ret = conn->c_trans->xmit(conn, rm,  in rds_send_xmit()
305 						  conn->c_xmit_hdr_off,  in rds_send_xmit()
306 						  conn->c_xmit_sg,  in rds_send_xmit()
307 						  conn->c_xmit_data_off);  in rds_send_xmit()
311 			if (conn->c_xmit_hdr_off < sizeof(struct rds_header)) {  in rds_send_xmit()
314 					    conn->c_xmit_hdr_off);  in rds_send_xmit()
315 				conn->c_xmit_hdr_off += tmp;  in rds_send_xmit()
319 			sg = &rm->data.op_sg[conn->c_xmit_sg];  in rds_send_xmit()
322 						      conn->c_xmit_data_off);  in rds_send_xmit()
323 				conn->c_xmit_data_off += tmp;  in rds_send_xmit()
325 				if (conn->c_xmit_data_off == sg->length) {  in rds_send_xmit()
326 					conn->c_xmit_data_off = 0;  in rds_send_xmit()
328 					conn->c_xmit_sg++;  in rds_send_xmit()
330 					       conn->c_xmit_sg == rm->data.op_nents);  in rds_send_xmit()
334 			if (conn->c_xmit_hdr_off == sizeof(struct rds_header) &&  in rds_send_xmit()
335 			    (conn->c_xmit_sg == rm->data.op_nents))  in rds_send_xmit()
336 				conn->c_xmit_data_sent = 1;  in rds_send_xmit()
344 		if (!rm->data.op_active || conn->c_xmit_data_sent) {  in rds_send_xmit()
345 			conn->c_xmit_rm = NULL;  in rds_send_xmit()
346 			conn->c_xmit_sg = 0;  in rds_send_xmit()
347 			conn->c_xmit_hdr_off = 0;  in rds_send_xmit()
348 			conn->c_xmit_data_off = 0;  in rds_send_xmit()
349 			conn->c_xmit_rdma_sent = 0;  in rds_send_xmit()
350 			conn->c_xmit_atomic_sent = 0;  in rds_send_xmit()
351 			conn->c_xmit_data_sent = 0;  in rds_send_xmit()
357 	if (conn->c_trans->xmit_complete)  in rds_send_xmit()
358 		conn->c_trans->xmit_complete(conn);  in rds_send_xmit()
360 	release_in_xmit(conn);  in rds_send_xmit()
383 		if (!list_empty(&conn->c_send_queue)) {  in rds_send_xmit()
521 struct rds_message *rds_send_get_message(struct rds_connection *conn,  in rds_send_get_message()  argument
527 	spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_get_message()
529 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_get_message()
537 	list_for_each_entry_safe(rm, tmp, &conn->c_send_queue, m_conn_item) {  in rds_send_get_message()
546 	spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_get_message()
642 void rds_send_drop_acked(struct rds_connection *conn, u64 ack,  in rds_send_drop_acked()  argument
649 	spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_drop_acked()
651 	list_for_each_entry_safe(rm, tmp, &conn->c_retrans, m_conn_item) {  in rds_send_drop_acked()
663 	spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_drop_acked()
673 	struct rds_connection *conn;  in rds_send_drop_to()  local
701 		conn = rm->m_inc.i_conn;  in rds_send_drop_to()
703 		spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_drop_to()
710 			spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_drop_to()
714 		spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_drop_to()
748 static int rds_send_queue_rm(struct rds_sock *rs, struct rds_connection *conn,  in rds_send_queue_rm()  argument
791 		rm->m_inc.i_conn = conn;  in rds_send_queue_rm()
794 		spin_lock(&conn->c_lock);  in rds_send_queue_rm()
795 		rm->m_inc.i_hdr.h_sequence = cpu_to_be64(conn->c_next_tx_seq++);  in rds_send_queue_rm()
796 		list_add_tail(&rm->m_conn_item, &conn->c_send_queue);  in rds_send_queue_rm()
798 		spin_unlock(&conn->c_lock);  in rds_send_queue_rm()
926 	struct rds_connection *conn;  in rds_sendmsg()  local
991 		conn = rs->rs_conn;  in rds_sendmsg()
993 		conn = rds_conn_create_outgoing(rs->rs_bound_addr, daddr,  in rds_sendmsg()
996 		if (IS_ERR(conn)) {  in rds_sendmsg()
997 			ret = PTR_ERR(conn);  in rds_sendmsg()
1000 		rs->rs_conn = conn;  in rds_sendmsg()
1008 	if (rm->rdma.op_active && !conn->c_trans->xmit_rdma) {  in rds_sendmsg()
1011 			       &rm->rdma, conn->c_trans->xmit_rdma);  in rds_sendmsg()
1016 	if (rm->atomic.op_active && !conn->c_trans->xmit_atomic) {  in rds_sendmsg()
1019 			       &rm->atomic, conn->c_trans->xmit_atomic);  in rds_sendmsg()
1024 	rds_conn_connect_if_down(conn);  in rds_sendmsg()
1026 	ret = rds_cong_wait(conn->c_fcong, dport, nonblock, rs);  in rds_sendmsg()
1032 	while (!rds_send_queue_rm(rs, conn, rm, rs->rs_bound_port,  in rds_sendmsg()
1046 					rds_send_queue_rm(rs, conn, rm,  in rds_sendmsg()
1067 	if (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags))  in rds_sendmsg()
1068 		rds_send_xmit(conn);  in rds_sendmsg()
1089 rds_send_pong(struct rds_connection *conn, __be16 dport)  in rds_send_pong()  argument
1101 	rm->m_daddr = conn->c_faddr;  in rds_send_pong()
1104 	rds_conn_connect_if_down(conn);  in rds_send_pong()
1106 	ret = rds_cong_wait(conn->c_fcong, dport, 1, NULL);  in rds_send_pong()
1110 	spin_lock_irqsave(&conn->c_lock, flags);  in rds_send_pong()
1111 	list_add_tail(&rm->m_conn_item, &conn->c_send_queue);  in rds_send_pong()
1114 	rm->m_inc.i_conn = conn;  in rds_send_pong()
1117 				    conn->c_next_tx_seq);  in rds_send_pong()
1118 	conn->c_next_tx_seq++;  in rds_send_pong()
1119 	spin_unlock_irqrestore(&conn->c_lock, flags);  in rds_send_pong()
1124 	if (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags))  in rds_send_pong()
1125 		rds_send_xmit(conn);  in rds_send_pong()