73 static void rds_ib_send_unmap_data(struct rds_ib_connection *ic,  in rds_ib_send_unmap_data()  argument
78 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_data()
83 static void rds_ib_send_unmap_rdma(struct rds_ib_connection *ic,  in rds_ib_send_unmap_rdma()  argument
88 		ib_dma_unmap_sg(ic->i_cm_id->device,  in rds_ib_send_unmap_rdma()
123 static void rds_ib_send_unmap_atomic(struct rds_ib_connection *ic,  in rds_ib_send_unmap_atomic()  argument
129 		ib_dma_unmap_sg(ic->i_cm_id->device, op->op_sg, 1,  in rds_ib_send_unmap_atomic()
150 static struct rds_message *rds_ib_send_unmap_op(struct rds_ib_connection *ic,  in rds_ib_send_unmap_op()  argument
161 			rds_ib_send_unmap_data(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
168 			rds_ib_send_unmap_rdma(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
175 			rds_ib_send_unmap_atomic(ic, send->s_op, wc_status);  in rds_ib_send_unmap_op()
190 void rds_ib_send_init_ring(struct rds_ib_connection *ic)  in rds_ib_send_init_ring()  argument
195 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_init_ring()
205 		sge->addr = ic->i_send_hdrs_dma[i];  in rds_ib_send_init_ring()
208 		sge->lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
210 		send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_send_init_ring()
214 void rds_ib_send_clear_ring(struct rds_ib_connection *ic)  in rds_ib_send_clear_ring()  argument
219 	for (i = 0, send = ic->i_sends; i < ic->i_send_ring.w_nr; i++, send++) {  in rds_ib_send_clear_ring()
221 			rds_ib_send_unmap_op(ic, send, IB_WC_WR_FLUSH_ERR);  in rds_ib_send_clear_ring()
229 static void rds_ib_sub_signaled(struct rds_ib_connection *ic, int nr)  in rds_ib_sub_signaled()  argument
231 	if ((atomic_sub_return(nr, &ic->i_signaled_sends) == 0) &&  in rds_ib_sub_signaled()
234 	BUG_ON(atomic_read(&ic->i_signaled_sends) < 0);  in rds_ib_sub_signaled()
243 void rds_ib_send_cqe_handler(struct rds_ib_connection *ic, struct ib_wc *wc)  in rds_ib_send_cqe_handler()  argument
246 	struct rds_connection *conn = ic->conn;  in rds_ib_send_cqe_handler()
261 		if (time_after(jiffies, ic->i_ack_queued + HZ / 2))  in rds_ib_send_cqe_handler()
263 		rds_ib_ack_send_complete(ic);  in rds_ib_send_cqe_handler()
267 	oldest = rds_ib_ring_oldest(&ic->i_send_ring);  in rds_ib_send_cqe_handler()
269 	completed = rds_ib_ring_completed(&ic->i_send_ring, wc->wr_id, oldest);  in rds_ib_send_cqe_handler()
272 		send = &ic->i_sends[oldest];  in rds_ib_send_cqe_handler()
276 		rm = rds_ib_send_unmap_op(ic, send, wc->status);  in rds_ib_send_cqe_handler()
292 		oldest = (oldest + 1) % ic->i_send_ring.w_nr;  in rds_ib_send_cqe_handler()
295 	rds_ib_ring_free(&ic->i_send_ring, completed);  in rds_ib_send_cqe_handler()
296 	rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_send_cqe_handler()
355 int rds_ib_send_grab_credits(struct rds_ib_connection *ic,  in rds_ib_send_grab_credits()  argument
362 	if (!ic->i_flowctl)  in rds_ib_send_grab_credits()
367 	oldval = newval = atomic_read(&ic->i_credits);  in rds_ib_send_grab_credits()
379 		struct rds_connection *conn = ic->i_cm_id->context;  in rds_ib_send_grab_credits()
401 	if (atomic_cmpxchg(&ic->i_credits, oldval, newval) != oldval)  in rds_ib_send_grab_credits()
410 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_send_add_credits()  local
417 			IB_GET_SEND_CREDITS(atomic_read(&ic->i_credits)),  in rds_ib_send_add_credits()
420 	atomic_add(IB_SET_SEND_CREDITS(credits), &ic->i_credits);  in rds_ib_send_add_credits()
431 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_advertise_credits()  local
436 	atomic_add(IB_SET_POST_CREDITS(posted), &ic->i_credits);  in rds_ib_advertise_credits()
450 	if (IB_GET_POST_CREDITS(atomic_read(&ic->i_credits)) >= 16)  in rds_ib_advertise_credits()
451 		set_bit(IB_ACK_REQUESTED, &ic->i_ack_flags);  in rds_ib_advertise_credits()
454 static inline int rds_ib_set_wr_signal_state(struct rds_ib_connection *ic,  in rds_ib_set_wr_signal_state()  argument
463 	if (ic->i_unsignaled_wrs-- == 0 || notify) {  in rds_ib_set_wr_signal_state()
464 		ic->i_unsignaled_wrs = rds_ib_sysctl_max_unsig_wrs;  in rds_ib_set_wr_signal_state()
487 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit()  local
488 	struct ib_device *dev = ic->i_cm_id->device;  in rds_ib_xmit()
524 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit()
532 	if (ic->i_flowctl) {  in rds_ib_xmit()
533 		credit_alloc = rds_ib_send_grab_credits(ic, work_alloc, &posted, 0, RDS_MAX_ADV_CREDIT);  in rds_ib_xmit()
536 			rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - credit_alloc);  in rds_ib_xmit()
549 	if (!ic->i_data_op) {  in rds_ib_xmit()
555 			rdsdebug("ic %p mapping rm %p: %d\n", ic, rm, rm->data.op_count);  in rds_ib_xmit()
558 				rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
569 		ic->i_data_op = &rm->data;  in rds_ib_xmit()
596 		rm->m_inc.i_hdr.h_ack = cpu_to_be64(rds_ib_piggyb_ack(ic));  in rds_ib_xmit()
602 		if (ic->i_flowctl) {  in rds_ib_xmit()
603 			rds_ib_send_grab_credits(ic, 0, &posted, 1, RDS_MAX_ADV_CREDIT - adv_credits);  in rds_ib_xmit()
619 	send = &ic->i_sends[pos];  in rds_ib_xmit()
622 	scat = &ic->i_data_op->op_sg[rm->data.op_dmasg];  in rds_ib_xmit()
635 		send->s_sge[0].addr = ic->i_send_hdrs_dma[pos];  in rds_ib_xmit()
638 		send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit()
640 		ib_dma_sync_single_for_cpu(ic->rds_ibdev->dev,  in rds_ib_xmit()
641 					   ic->i_send_hdrs_dma[pos],  in rds_ib_xmit()
644 		memcpy(ic->i_send_hdrs[pos], &rm->m_inc.i_hdr,  in rds_ib_xmit()
658 			send->s_sge[1].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit()
669 		rds_ib_set_wr_signal_state(ic, send, false);  in rds_ib_xmit()
674 		if (ic->i_flowctl && flow_controlled && i == (work_alloc - 1)) {  in rds_ib_xmit()
675 			rds_ib_set_wr_signal_state(ic, send, true);  in rds_ib_xmit()
685 		if (ic->i_flowctl && adv_credits) {  in rds_ib_xmit()
686 			struct rds_header *hdr = ic->i_send_hdrs[pos];  in rds_ib_xmit()
694 		ib_dma_sync_single_for_device(ic->rds_ibdev->dev,  in rds_ib_xmit()
695 					      ic->i_send_hdrs_dma[pos],  in rds_ib_xmit()
703 		pos = (pos + 1) % ic->i_send_ring.w_nr;  in rds_ib_xmit()
704 		send = &ic->i_sends[pos];  in rds_ib_xmit()
717 		prev->s_op = ic->i_data_op;  in rds_ib_xmit()
720 			nr_sig += rds_ib_set_wr_signal_state(ic, prev, true);  in rds_ib_xmit()
721 		ic->i_data_op = NULL;  in rds_ib_xmit()
726 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit()
729 	if (ic->i_flowctl && i < credit_alloc)  in rds_ib_xmit()
733 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit()
737 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_wr, &failed_wr);  in rds_ib_xmit()
738 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit()
744 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit()
745 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit()
747 			ic->i_data_op = prev->s_op;  in rds_ib_xmit()
751 		rds_ib_conn_error(ic->conn, "ib_post_send failed\n");  in rds_ib_xmit()
768 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_atomic()  local
776 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, 1, &pos);  in rds_ib_xmit_atomic()
784 	send = &ic->i_sends[pos];  in rds_ib_xmit_atomic()
801 	nr_sig = rds_ib_set_wr_signal_state(ic, send, op->op_notify);  in rds_ib_xmit_atomic()
810 	ret = ib_dma_map_sg(ic->i_cm_id->device, op->op_sg, 1, DMA_FROM_DEVICE);  in rds_ib_xmit_atomic()
811 	rdsdebug("ic %p mapping atomic op %p. mapped %d pg\n", ic, op, ret);  in rds_ib_xmit_atomic()
813 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
822 	send->s_sge[0].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_atomic()
828 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_atomic()
831 	ret = ib_post_send(ic->i_cm_id->qp, &send->s_atomic_wr.wr, &failed_wr);  in rds_ib_xmit_atomic()
832 	rdsdebug("ic %p send %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_atomic()
838 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_atomic()
839 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_atomic()
854 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_rdma()  local
862 	u32 max_sge = ic->rds_ibdev->max_sge;  in rds_ib_xmit_rdma()
878 				ib_dma_map_sg(ic->i_cm_id->device, op->op_sg,  in rds_ib_xmit_rdma()
882 			rdsdebug("ic %p mapping op %p: %d\n", ic, op,  in rds_ib_xmit_rdma()
902 	work_alloc = rds_ib_ring_alloc(&ic->i_send_ring, i, &pos);  in rds_ib_xmit_rdma()
904 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
910 	send = &ic->i_sends[pos];  in rds_ib_xmit_rdma()
923 			nr_sig += rds_ib_set_wr_signal_state(ic, send,  in rds_ib_xmit_rdma()
947 				send->s_sge[j].lkey = ic->i_pd->local_dma_lkey;  in rds_ib_xmit_rdma()
955 			rdsdebug("ic %p sent %d remote_addr %llu\n", ic, sent, remote_addr);  in rds_ib_xmit_rdma()
968 		if (++send == &ic->i_sends[ic->i_send_ring.w_nr])  in rds_ib_xmit_rdma()
969 			send = ic->i_sends;  in rds_ib_xmit_rdma()
979 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc - i);  in rds_ib_xmit_rdma()
984 		atomic_add(nr_sig, &ic->i_signaled_sends);  in rds_ib_xmit_rdma()
987 	ret = ib_post_send(ic->i_cm_id->qp, &first->s_rdma_wr.wr, &failed_wr);  in rds_ib_xmit_rdma()
988 	rdsdebug("ic %p first %p (wr %p) ret %d wr %p\n", ic,  in rds_ib_xmit_rdma()
994 		rds_ib_ring_unalloc(&ic->i_send_ring, work_alloc);  in rds_ib_xmit_rdma()
995 		rds_ib_sub_signaled(ic, nr_sig);  in rds_ib_xmit_rdma()
1012 	struct rds_ib_connection *ic = conn->c_transport_data;  in rds_ib_xmit_path_complete()  local
1016 	rds_ib_attempt_ack(ic);  in rds_ib_xmit_path_complete()