66 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt);
67 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt);
68 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,
70 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt);
71 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt);
73 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt);
74 static void rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt);
75 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt);
87 static void rpcrdma_xprt_drain(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_drain()  argument
89 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_drain()
128 void rpcrdma_flush_disconnect(struct rpcrdma_xprt *r_xprt, struct ib_wc *wc)  in rpcrdma_flush_disconnect()  argument
131 		rpcrdma_force_disconnect(r_xprt->rx_ep);  in rpcrdma_flush_disconnect()
145 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_send()  local
149 	rpcrdma_sendctx_put_locked(r_xprt, sc);  in rpcrdma_wc_send()
150 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_send()
164 	struct rpcrdma_xprt *r_xprt = cq->cq_context;  in rpcrdma_wc_receive()  local
168 	--r_xprt->rx_ep->re_receive_count;  in rpcrdma_wc_receive()
185 	rpcrdma_flush_disconnect(r_xprt, wc);  in rpcrdma_wc_receive()
186 	rpcrdma_rep_put(&r_xprt->rx_buf, rep);  in rpcrdma_wc_receive()
283 static struct rdma_cm_id *rpcrdma_create_id(struct rpcrdma_xprt *r_xprt,  in rpcrdma_create_id()  argument
287 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_create_id()
367 static int rpcrdma_ep_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_ep_create()  argument
378 	ep->re_xprt = &r_xprt->rx_xprt;  in rpcrdma_ep_create()
381 	id = rpcrdma_create_id(r_xprt, ep);  in rpcrdma_ep_create()
391 	ep->re_max_requests = r_xprt->rx_xprt.max_reqs;  in rpcrdma_ep_create()
398 	r_xprt->rx_buf.rb_max_requests = cpu_to_be32(ep->re_max_requests);  in rpcrdma_ep_create()
410 	ep->re_attr.send_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
419 	ep->re_attr.recv_cq = ib_alloc_cq_any(device, r_xprt,  in rpcrdma_ep_create()
471 	r_xprt->rx_ep = ep;  in rpcrdma_ep_create()
486 int rpcrdma_xprt_connect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_connect()  argument
488 	struct rpc_xprt *xprt = &r_xprt->rx_xprt;  in rpcrdma_xprt_connect()
492 	rc = rpcrdma_ep_create(r_xprt);  in rpcrdma_xprt_connect()
495 	ep = r_xprt->rx_ep;  in rpcrdma_xprt_connect()
498 	rpcrdma_reset_cwnd(r_xprt);  in rpcrdma_xprt_connect()
504 	rpcrdma_post_recvs(r_xprt, 1, true);  in rpcrdma_xprt_connect()
519 	rc = rpcrdma_sendctxs_create(r_xprt);  in rpcrdma_xprt_connect()
525 	rc = rpcrdma_reqs_setup(r_xprt);  in rpcrdma_xprt_connect()
530 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_xprt_connect()
531 	frwr_wp_create(r_xprt);  in rpcrdma_xprt_connect()
534 	trace_xprtrdma_connect(r_xprt, rc);  in rpcrdma_xprt_connect()
548 void rpcrdma_xprt_disconnect(struct rpcrdma_xprt *r_xprt)  in rpcrdma_xprt_disconnect()  argument
550 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_xprt_disconnect()
559 	trace_xprtrdma_disconnect(r_xprt, rc);  in rpcrdma_xprt_disconnect()
561 	rpcrdma_xprt_drain(r_xprt);  in rpcrdma_xprt_disconnect()
562 	rpcrdma_reps_unmap(r_xprt);  in rpcrdma_xprt_disconnect()
563 	rpcrdma_reqs_reset(r_xprt);  in rpcrdma_xprt_disconnect()
564 	rpcrdma_mrs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
565 	rpcrdma_sendctxs_destroy(r_xprt);  in rpcrdma_xprt_disconnect()
570 	r_xprt->rx_ep = NULL;  in rpcrdma_xprt_disconnect()
590 static void rpcrdma_sendctxs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_destroy()  argument
592 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_destroy()
619 static int rpcrdma_sendctxs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctxs_create()  argument
621 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctxs_create()
630 	i = r_xprt->rx_ep->re_max_requests + RPCRDMA_MAX_BC_REQUESTS;  in rpcrdma_sendctxs_create()
637 		sc = rpcrdma_sendctx_create(r_xprt->rx_ep);  in rpcrdma_sendctxs_create()
672 struct rpcrdma_sendctx *rpcrdma_sendctx_get_locked(struct rpcrdma_xprt *r_xprt)  in rpcrdma_sendctx_get_locked()  argument
674 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_get_locked()
698 	xprt_wait_for_buffer_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_get_locked()
699 	r_xprt->rx_stats.empty_sendctx_q++;  in rpcrdma_sendctx_get_locked()
713 static void rpcrdma_sendctx_put_locked(struct rpcrdma_xprt *r_xprt,  in rpcrdma_sendctx_put_locked()  argument
716 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_sendctx_put_locked()
734 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_sendctx_put_locked()
738 rpcrdma_mrs_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_create()  argument
740 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_create()
741 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_create()
755 		rc = frwr_mr_init(r_xprt, mr);  in rpcrdma_mrs_create()
767 	r_xprt->rx_stats.mrs_allocated += count;  in rpcrdma_mrs_create()
768 	trace_xprtrdma_createmrs(r_xprt, count);  in rpcrdma_mrs_create()
776 	struct rpcrdma_xprt *r_xprt = container_of(buf, struct rpcrdma_xprt,  in rpcrdma_mr_refresh_worker()  local
779 	rpcrdma_mrs_create(r_xprt);  in rpcrdma_mr_refresh_worker()
780 	xprt_write_space(&r_xprt->rx_xprt);  in rpcrdma_mr_refresh_worker()
788 void rpcrdma_mrs_refresh(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_refresh()  argument
790 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_refresh()
791 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_mrs_refresh()
808 struct rpcrdma_req *rpcrdma_req_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_req_create()  argument
811 	struct rpcrdma_buffer *buffer = &r_xprt->rx_buf;  in rpcrdma_req_create()
848 int rpcrdma_req_setup(struct rpcrdma_xprt *r_xprt, struct rpcrdma_req *req)  in rpcrdma_req_setup()  argument
855 		     r_xprt->rx_ep->re_max_rdma_segs * rpcrdma_readchunk_maxsz;  in rpcrdma_req_setup()
862 	if (!__rpcrdma_regbuf_dma_map(r_xprt, rb))  in rpcrdma_req_setup()
880 static int rpcrdma_reqs_setup(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_setup()  argument
882 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_setup()
887 		rc = rpcrdma_req_setup(r_xprt, req);  in rpcrdma_reqs_setup()
913 static void rpcrdma_reqs_reset(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reqs_reset()  argument
915 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reqs_reset()
923 struct rpcrdma_rep *rpcrdma_rep_create(struct rpcrdma_xprt *r_xprt,  in rpcrdma_rep_create()  argument
926 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_rep_create()
933 	rep->rr_rdmabuf = rpcrdma_regbuf_alloc(r_xprt->rx_ep->re_inline_recv,  in rpcrdma_rep_create()
939 		atomic_inc_return(&r_xprt->rx_ep->re_completion_ids);  in rpcrdma_rep_create()
944 	rep->rr_rxprt = r_xprt;  in rpcrdma_rep_create()
1005 static void rpcrdma_reps_unmap(struct rpcrdma_xprt *r_xprt)  in rpcrdma_reps_unmap()  argument
1007 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_reps_unmap()
1040 int rpcrdma_buffer_create(struct rpcrdma_xprt *r_xprt)  in rpcrdma_buffer_create()  argument
1042 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_buffer_create()
1056 	for (i = 0; i < r_xprt->rx_xprt.max_reqs; i++) {  in rpcrdma_buffer_create()
1059 		req = rpcrdma_req_create(r_xprt,  in rpcrdma_buffer_create()
1110 static void rpcrdma_mrs_destroy(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mrs_destroy()  argument
1112 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mrs_destroy()
1163 rpcrdma_mr_get(struct rpcrdma_xprt *r_xprt)  in rpcrdma_mr_get()  argument
1165 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_mr_get()
1281 bool __rpcrdma_regbuf_dma_map(struct rpcrdma_xprt *r_xprt,  in __rpcrdma_regbuf_dma_map()  argument
1284 	struct ib_device *device = r_xprt->rx_ep->re_id->device;  in __rpcrdma_regbuf_dma_map()
1297 	rb->rg_iov.lkey = r_xprt->rx_ep->re_pd->local_dma_lkey;  in __rpcrdma_regbuf_dma_map()
1329 void rpcrdma_post_recvs(struct rpcrdma_xprt *r_xprt, int needed, bool temp)  in rpcrdma_post_recvs()  argument
1331 	struct rpcrdma_buffer *buf = &r_xprt->rx_buf;  in rpcrdma_post_recvs()
1332 	struct rpcrdma_ep *ep = r_xprt->rx_ep;  in rpcrdma_post_recvs()
1358 			rep = rpcrdma_rep_create(r_xprt, temp);  in rpcrdma_post_recvs()
1361 		if (!rpcrdma_regbuf_dma_map(r_xprt, rep->rr_rdmabuf)) {  in rpcrdma_post_recvs()
1379 		trace_xprtrdma_post_recvs_err(r_xprt, rc);  in rpcrdma_post_recvs()
1393 	trace_xprtrdma_post_recvs(r_xprt, count);  in rpcrdma_post_recvs()