591 			struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last);  in rvt_clear_mr_refs()  local
593 			rvt_put_qp_swqe(qp, wqe);  in rvt_clear_mr_refs()
621 static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey)  in rvt_swqe_has_lkey()  argument
625 	for (i = 0; i < wqe->wr.num_sge; i++) {  in rvt_swqe_has_lkey()
626 		struct rvt_sge *sge = &wqe->sg_list[i];  in rvt_swqe_has_lkey()
644 		struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last);  in rvt_qp_sends_has_lkey()  local
646 		if (rvt_swqe_has_lkey(wqe, lkey))  in rvt_qp_sends_has_lkey()
979 	struct rvt_swqe *wqe;  in free_ud_wq_attr()  local
983 		wqe = rvt_get_swqe_ptr(qp, i);  in free_ud_wq_attr()
984 		kfree(wqe->ud_wr.attr);  in free_ud_wq_attr()
985 		wqe->ud_wr.attr = NULL;  in free_ud_wq_attr()
999 	struct rvt_swqe *wqe;  in alloc_ud_wq_attr()  local
1003 		wqe = rvt_get_swqe_ptr(qp, i);  in alloc_ud_wq_attr()
1004 		wqe->ud_wr.attr = kzalloc_node(sizeof(*wqe->ud_wr.attr),  in alloc_ud_wq_attr()
1006 		if (!wqe->ud_wr.attr) {  in alloc_ud_wq_attr()
1797 		struct rvt_rwqe *wqe;  in rvt_post_recv()  local
1825 			wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head);  in rvt_post_recv()
1826 			wqe->wr_id = wr->wr_id;  in rvt_post_recv()
1827 			wqe->num_sge = wr->num_sge;  in rvt_post_recv()
1829 				wqe->sg_list[i].addr = wr->sg_list[i].addr;  in rvt_post_recv()
1830 				wqe->sg_list[i].length = wr->sg_list[i].length;  in rvt_post_recv()
1831 				wqe->sg_list[i].lkey = wr->sg_list[i].lkey;  in rvt_post_recv()
1964 	struct rvt_swqe *wqe;  in rvt_post_one_wr()  local
2038 	wqe = rvt_get_swqe_ptr(qp, qp->s_head);  in rvt_post_one_wr()
2041 	memcpy(&wqe->ud_wr, wr, cplen);  in rvt_post_one_wr()
2043 	wqe->length = 0;  in rvt_post_one_wr()
2055 			ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge,  in rvt_post_one_wr()
2059 			wqe->length += length;  in rvt_post_one_wr()
2061 				last_sge = &wqe->sg_list[j];  in rvt_post_one_wr()
2064 		wqe->wr.num_sge = j;  in rvt_post_one_wr()
2074 		struct rvt_ah *ah = rvt_get_swqe_ah(wqe);  in rvt_post_one_wr()
2077 		rdma_copy_ah_attr(wqe->ud_wr.attr, &ah->attr);  in rvt_post_one_wr()
2084 			wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY;  in rvt_post_one_wr()
2085 		wqe->ssn = 0;  in rvt_post_one_wr()
2086 		wqe->psn = 0;  in rvt_post_one_wr()
2087 		wqe->lpsn = 0;  in rvt_post_one_wr()
2089 		wqe->ssn = qp->s_ssn++;  in rvt_post_one_wr()
2090 		wqe->psn = qp->s_next_psn;  in rvt_post_one_wr()
2091 		wqe->lpsn = wqe->psn +  in rvt_post_one_wr()
2092 				(wqe->length ?  in rvt_post_one_wr()
2093 					((wqe->length - 1) >> log_pmtu) :  in rvt_post_one_wr()
2099 		ret = rdi->driver_f.setup_wqe(qp, wqe, call_send);  in rvt_post_one_wr()
2105 		qp->s_next_psn = wqe->lpsn + 1;  in rvt_post_one_wr()
2108 		wqe->wr.send_flags |= RVT_SEND_RESERVE_USED;  in rvt_post_one_wr()
2109 		rvt_qp_wqe_reserve(qp, wqe);  in rvt_post_one_wr()
2111 		wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED;  in rvt_post_one_wr()
2114 	trace_rvt_post_one_wr(qp, wqe, wr->num_sge);  in rvt_post_one_wr()
2122 		rdma_destroy_ah_attr(wqe->ud_wr.attr);  in rvt_post_one_wr()
2126 		struct rvt_sge *sge = &wqe->sg_list[--j];  in rvt_post_one_wr()
2212 		struct rvt_rwqe *wqe;  in rvt_post_srq_recv()  local
2232 		wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head);  in rvt_post_srq_recv()
2233 		wqe->wr_id = wr->wr_id;  in rvt_post_srq_recv()
2234 		wqe->num_sge = wr->num_sge;  in rvt_post_srq_recv()
2236 			wqe->sg_list[i].addr = wr->sg_list[i].addr;  in rvt_post_srq_recv()
2237 			wqe->sg_list[i].length = wr->sg_list[i].length;  in rvt_post_srq_recv()
2238 			wqe->sg_list[i].lkey = wr->sg_list[i].lkey;  in rvt_post_srq_recv()
2267 static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe)  in init_sge()  argument
2281 	for (i = j = 0; i < wqe->num_sge; i++) {  in init_sge()
2282 		if (wqe->sg_list[i].length == 0)  in init_sge()
2286 				  NULL, rvt_cast_sge(&wqe->sg_list[i]),  in init_sge()
2290 		qp->r_len += wqe->sg_list[i].length;  in init_sge()
2305 	wc.wr_id = wqe->wr_id;  in init_sge()
2350 	struct rvt_rwqe *wqe;  in rvt_get_rwqe()  local
2396 	wqe = rvt_get_rwqe_ptr(rq, tail);  in rvt_get_rwqe()
2408 	if (!wr_id_only && !init_sge(qp, wqe)) {  in rvt_get_rwqe()
2412 	qp->r_wr_id = wqe->wr_id;  in rvt_get_rwqe()
2778 void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe,  in rvt_send_complete()  argument
2789 	trace_rvt_qp_send_completion(qp, wqe, old_last);  in rvt_send_complete()
2790 	last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode],  in rvt_send_complete()
2907 	struct rvt_swqe *wqe;  in rvt_ruc_loopback()  local
2942 	wqe = rvt_get_swqe_ptr(sqp, sqp->s_last);  in rvt_ruc_loopback()
2979 	sqp->s_sge.sge = wqe->sg_list[0];  in rvt_ruc_loopback()
2980 	sqp->s_sge.sg_list = wqe->sg_list + 1;  in rvt_ruc_loopback()
2981 	sqp->s_sge.num_sge = wqe->wr.num_sge;  in rvt_ruc_loopback()
2982 	sqp->s_len = wqe->length;  in rvt_ruc_loopback()
2983 	switch (wqe->wr.opcode) {  in rvt_ruc_loopback()
2988 		if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {  in rvt_ruc_loopback()
2990 						wqe->wr.ex.invalidate_rkey))  in rvt_ruc_loopback()
3004 		if (wqe->length > qp->r_len)  in rvt_ruc_loopback()
3006 		switch (wqe->wr.opcode) {  in rvt_ruc_loopback()
3009 						 wqe->wr.ex.invalidate_rkey)) {  in rvt_ruc_loopback()
3012 					wqe->wr.ex.invalidate_rkey;  in rvt_ruc_loopback()
3017 			wc.ex.imm_data = wqe->wr.ex.imm_data;  in rvt_ruc_loopback()
3028 		wc.ex.imm_data = wqe->wr.ex.imm_data;  in rvt_ruc_loopback()
3041 		if (wqe->length == 0)  in rvt_ruc_loopback()
3043 		if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length,  in rvt_ruc_loopback()
3044 					  wqe->rdma_wr.remote_addr,  in rvt_ruc_loopback()
3045 					  wqe->rdma_wr.rkey,  in rvt_ruc_loopback()
3050 		qp->r_sge.total_len = wqe->length;  in rvt_ruc_loopback()
3056 		if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length,  in rvt_ruc_loopback()
3057 					  wqe->rdma_wr.remote_addr,  in rvt_ruc_loopback()
3058 					  wqe->rdma_wr.rkey,  in rvt_ruc_loopback()
3064 		qp->r_sge.sge = wqe->sg_list[0];  in rvt_ruc_loopback()
3065 		qp->r_sge.sg_list = wqe->sg_list + 1;  in rvt_ruc_loopback()
3066 		qp->r_sge.num_sge = wqe->wr.num_sge;  in rvt_ruc_loopback()
3067 		qp->r_sge.total_len = wqe->length;  in rvt_ruc_loopback()
3074 		if (unlikely(wqe->atomic_wr.remote_addr & (sizeof(u64) - 1)))  in rvt_ruc_loopback()
3077 					  wqe->atomic_wr.remote_addr,  in rvt_ruc_loopback()
3078 					  wqe->atomic_wr.rkey,  in rvt_ruc_loopback()
3083 		sdata = wqe->atomic_wr.compare_add;  in rvt_ruc_loopback()
3085 			(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?  in rvt_ruc_loopback()
3088 				      sdata, wqe->atomic_wr.swap);  in rvt_ruc_loopback()
3114 	if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)  in rvt_ruc_loopback()
3120 	wc.byte_len = wqe->length;  in rvt_ruc_loopback()
3127 	rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED);  in rvt_ruc_loopback()
3136 	rvt_send_complete(sqp, wqe, send_status);  in rvt_ruc_loopback()
3192 	rvt_send_complete(sqp, wqe, send_status);  in rvt_ruc_loopback()