118 static void svc_rdma_recv_cid_init(struct svcxprt_rdma *rdma,  in svc_rdma_recv_cid_init()  argument
121 	cid->ci_queue_id = rdma->sc_rq_cq->res.id;  in svc_rdma_recv_cid_init()
122 	cid->ci_completion_id = atomic_inc_return(&rdma->sc_completion_ids);  in svc_rdma_recv_cid_init()
126 svc_rdma_recv_ctxt_alloc(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxt_alloc()  argument
135 	buffer = kmalloc(rdma->sc_max_req_size, GFP_KERNEL);  in svc_rdma_recv_ctxt_alloc()
138 	addr = ib_dma_map_single(rdma->sc_pd->device, buffer,  in svc_rdma_recv_ctxt_alloc()
139 				 rdma->sc_max_req_size, DMA_FROM_DEVICE);  in svc_rdma_recv_ctxt_alloc()
140 	if (ib_dma_mapping_error(rdma->sc_pd->device, addr))  in svc_rdma_recv_ctxt_alloc()
143 	svc_rdma_recv_cid_init(rdma, &ctxt->rc_cid);  in svc_rdma_recv_ctxt_alloc()
155 	ctxt->rc_recv_sge.length = rdma->sc_max_req_size;  in svc_rdma_recv_ctxt_alloc()
156 	ctxt->rc_recv_sge.lkey = rdma->sc_pd->local_dma_lkey;  in svc_rdma_recv_ctxt_alloc()
169 static void svc_rdma_recv_ctxt_destroy(struct svcxprt_rdma *rdma,  in svc_rdma_recv_ctxt_destroy()  argument
172 	ib_dma_unmap_single(rdma->sc_pd->device, ctxt->rc_recv_sge.addr,  in svc_rdma_recv_ctxt_destroy()
183 void svc_rdma_recv_ctxts_destroy(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxts_destroy()  argument
188 	while ((node = llist_del_first(&rdma->sc_recv_ctxts))) {  in svc_rdma_recv_ctxts_destroy()
190 		svc_rdma_recv_ctxt_destroy(rdma, ctxt);  in svc_rdma_recv_ctxts_destroy()
200 struct svc_rdma_recv_ctxt *svc_rdma_recv_ctxt_get(struct svcxprt_rdma *rdma)  in svc_rdma_recv_ctxt_get()  argument
205 	node = llist_del_first(&rdma->sc_recv_ctxts);  in svc_rdma_recv_ctxt_get()
215 	ctxt = svc_rdma_recv_ctxt_alloc(rdma);  in svc_rdma_recv_ctxt_get()
227 void svc_rdma_recv_ctxt_put(struct svcxprt_rdma *rdma,  in svc_rdma_recv_ctxt_put()  argument
236 		llist_add(&ctxt->rc_node, &rdma->sc_recv_ctxts);  in svc_rdma_recv_ctxt_put()
238 		svc_rdma_recv_ctxt_destroy(rdma, ctxt);  in svc_rdma_recv_ctxt_put()
253 	struct svcxprt_rdma *rdma =  in svc_rdma_release_rqst()  local
258 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_release_rqst()
261 static bool svc_rdma_refresh_recvs(struct svcxprt_rdma *rdma,  in svc_rdma_refresh_recvs()  argument
269 	if (test_bit(XPT_CLOSE, &rdma->sc_xprt.xpt_flags))  in svc_rdma_refresh_recvs()
274 		ctxt = svc_rdma_recv_ctxt_get(rdma);  in svc_rdma_refresh_recvs()
282 		rdma->sc_pending_recvs++;  in svc_rdma_refresh_recvs()
287 	ret = ib_post_recv(rdma->sc_qp, recv_chain, &bad_wr);  in svc_rdma_refresh_recvs()
293 	trace_svcrdma_rq_post_err(rdma, ret);  in svc_rdma_refresh_recvs()
298 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_refresh_recvs()
311 bool svc_rdma_post_recvs(struct svcxprt_rdma *rdma)  in svc_rdma_post_recvs()  argument
313 	return svc_rdma_refresh_recvs(rdma, rdma->sc_max_requests, true);  in svc_rdma_post_recvs()
324 	struct svcxprt_rdma *rdma = cq->cq_context;  in svc_rdma_wc_receive()  local
328 	rdma->sc_pending_recvs--;  in svc_rdma_wc_receive()
346 	if (rdma->sc_pending_recvs < rdma->sc_max_requests)  in svc_rdma_wc_receive()
347 		if (!svc_rdma_refresh_recvs(rdma, rdma->sc_recv_batch, false))  in svc_rdma_wc_receive()
353 	spin_lock(&rdma->sc_rq_dto_lock);  in svc_rdma_wc_receive()
354 	list_add_tail(&ctxt->rc_list, &rdma->sc_rq_dto_q);  in svc_rdma_wc_receive()
356 	set_bit(XPT_DATA, &rdma->sc_xprt.xpt_flags);  in svc_rdma_wc_receive()
357 	spin_unlock(&rdma->sc_rq_dto_lock);  in svc_rdma_wc_receive()
358 	if (!test_bit(RDMAXPRT_CONN_PENDING, &rdma->sc_flags))  in svc_rdma_wc_receive()
359 		svc_xprt_enqueue(&rdma->sc_xprt);  in svc_rdma_wc_receive()
368 	svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_wc_receive()
369 	svc_xprt_deferred_close(&rdma->sc_xprt);  in svc_rdma_wc_receive()
377 void svc_rdma_flush_recv_queues(struct svcxprt_rdma *rdma)  in svc_rdma_flush_recv_queues()  argument
381 	while ((ctxt = svc_rdma_next_recv_ctxt(&rdma->sc_rq_dto_q))) {  in svc_rdma_flush_recv_queues()
383 		svc_rdma_recv_ctxt_put(rdma, ctxt);  in svc_rdma_flush_recv_queues()
588 static void svc_rdma_get_inv_rkey(struct svcxprt_rdma *rdma,  in svc_rdma_get_inv_rkey()  argument
597 	if (!rdma->sc_snd_w_inv)  in svc_rdma_get_inv_rkey()
718 static void svc_rdma_send_error(struct svcxprt_rdma *rdma,  in svc_rdma_send_error()  argument
724 	sctxt = svc_rdma_send_ctxt_get(rdma);  in svc_rdma_send_error()
727 	svc_rdma_send_error_msg(rdma, sctxt, rctxt, status);  in svc_rdma_send_error()