Lines Matching refs:wqe
593 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, qp->s_last); in rvt_clear_mr_refs() local
595 rvt_put_qp_swqe(qp, wqe); in rvt_clear_mr_refs()
623 static bool rvt_swqe_has_lkey(struct rvt_swqe *wqe, u32 lkey) in rvt_swqe_has_lkey() argument
627 for (i = 0; i < wqe->wr.num_sge; i++) { in rvt_swqe_has_lkey()
628 struct rvt_sge *sge = &wqe->sg_list[i]; in rvt_swqe_has_lkey()
646 struct rvt_swqe *wqe = rvt_get_swqe_ptr(qp, s_last); in rvt_qp_sends_has_lkey() local
648 if (rvt_swqe_has_lkey(wqe, lkey)) in rvt_qp_sends_has_lkey()
981 struct rvt_swqe *wqe; in free_ud_wq_attr() local
985 wqe = rvt_get_swqe_ptr(qp, i); in free_ud_wq_attr()
986 kfree(wqe->ud_wr.attr); in free_ud_wq_attr()
987 wqe->ud_wr.attr = NULL; in free_ud_wq_attr()
1001 struct rvt_swqe *wqe; in alloc_ud_wq_attr() local
1005 wqe = rvt_get_swqe_ptr(qp, i); in alloc_ud_wq_attr()
1006 wqe->ud_wr.attr = kzalloc_node(sizeof(*wqe->ud_wr.attr), in alloc_ud_wq_attr()
1008 if (!wqe->ud_wr.attr) { in alloc_ud_wq_attr()
1799 struct rvt_rwqe *wqe; in rvt_post_recv() local
1827 wqe = rvt_get_rwqe_ptr(&qp->r_rq, wq->head); in rvt_post_recv()
1828 wqe->wr_id = wr->wr_id; in rvt_post_recv()
1829 wqe->num_sge = wr->num_sge; in rvt_post_recv()
1831 wqe->sg_list[i].addr = wr->sg_list[i].addr; in rvt_post_recv()
1832 wqe->sg_list[i].length = wr->sg_list[i].length; in rvt_post_recv()
1833 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; in rvt_post_recv()
1966 struct rvt_swqe *wqe; in rvt_post_one_wr() local
2040 wqe = rvt_get_swqe_ptr(qp, qp->s_head); in rvt_post_one_wr()
2043 memcpy(&wqe->wr, wr, cplen); in rvt_post_one_wr()
2045 wqe->length = 0; in rvt_post_one_wr()
2057 ret = rvt_lkey_ok(rkt, pd, &wqe->sg_list[j], last_sge, in rvt_post_one_wr()
2061 wqe->length += length; in rvt_post_one_wr()
2063 last_sge = &wqe->sg_list[j]; in rvt_post_one_wr()
2066 wqe->wr.num_sge = j; in rvt_post_one_wr()
2076 struct rvt_ah *ah = rvt_get_swqe_ah(wqe); in rvt_post_one_wr()
2079 rdma_copy_ah_attr(wqe->ud_wr.attr, &ah->attr); in rvt_post_one_wr()
2086 wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY; in rvt_post_one_wr()
2087 wqe->ssn = 0; in rvt_post_one_wr()
2088 wqe->psn = 0; in rvt_post_one_wr()
2089 wqe->lpsn = 0; in rvt_post_one_wr()
2091 wqe->ssn = qp->s_ssn++; in rvt_post_one_wr()
2092 wqe->psn = qp->s_next_psn; in rvt_post_one_wr()
2093 wqe->lpsn = wqe->psn + in rvt_post_one_wr()
2094 (wqe->length ? in rvt_post_one_wr()
2095 ((wqe->length - 1) >> log_pmtu) : in rvt_post_one_wr()
2101 ret = rdi->driver_f.setup_wqe(qp, wqe, call_send); in rvt_post_one_wr()
2107 qp->s_next_psn = wqe->lpsn + 1; in rvt_post_one_wr()
2110 wqe->wr.send_flags |= RVT_SEND_RESERVE_USED; in rvt_post_one_wr()
2111 rvt_qp_wqe_reserve(qp, wqe); in rvt_post_one_wr()
2113 wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED; in rvt_post_one_wr()
2116 trace_rvt_post_one_wr(qp, wqe, wr->num_sge); in rvt_post_one_wr()
2124 rdma_destroy_ah_attr(wqe->ud_wr.attr); in rvt_post_one_wr()
2128 struct rvt_sge *sge = &wqe->sg_list[--j]; in rvt_post_one_wr()
2214 struct rvt_rwqe *wqe; in rvt_post_srq_recv() local
2234 wqe = rvt_get_rwqe_ptr(&srq->rq, wq->head); in rvt_post_srq_recv()
2235 wqe->wr_id = wr->wr_id; in rvt_post_srq_recv()
2236 wqe->num_sge = wr->num_sge; in rvt_post_srq_recv()
2238 wqe->sg_list[i].addr = wr->sg_list[i].addr; in rvt_post_srq_recv()
2239 wqe->sg_list[i].length = wr->sg_list[i].length; in rvt_post_srq_recv()
2240 wqe->sg_list[i].lkey = wr->sg_list[i].lkey; in rvt_post_srq_recv()
2269 static int init_sge(struct rvt_qp *qp, struct rvt_rwqe *wqe) in init_sge() argument
2283 for (i = j = 0; i < wqe->num_sge; i++) { in init_sge()
2284 if (wqe->sg_list[i].length == 0) in init_sge()
2288 NULL, rvt_cast_sge(&wqe->sg_list[i]), in init_sge()
2292 qp->r_len += wqe->sg_list[i].length; in init_sge()
2307 wc.wr_id = wqe->wr_id; in init_sge()
2352 struct rvt_rwqe *wqe; in rvt_get_rwqe() local
2398 wqe = rvt_get_rwqe_ptr(rq, tail); in rvt_get_rwqe()
2410 if (!wr_id_only && !init_sge(qp, wqe)) { in rvt_get_rwqe()
2414 qp->r_wr_id = wqe->wr_id; in rvt_get_rwqe()
2780 void rvt_send_complete(struct rvt_qp *qp, struct rvt_swqe *wqe, in rvt_send_complete() argument
2791 trace_rvt_qp_send_completion(qp, wqe, old_last); in rvt_send_complete()
2792 last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode], in rvt_send_complete()
2909 struct rvt_swqe *wqe; in rvt_ruc_loopback() local
2944 wqe = rvt_get_swqe_ptr(sqp, sqp->s_last); in rvt_ruc_loopback()
2981 sqp->s_sge.sge = wqe->sg_list[0]; in rvt_ruc_loopback()
2982 sqp->s_sge.sg_list = wqe->sg_list + 1; in rvt_ruc_loopback()
2983 sqp->s_sge.num_sge = wqe->wr.num_sge; in rvt_ruc_loopback()
2984 sqp->s_len = wqe->length; in rvt_ruc_loopback()
2985 switch (wqe->wr.opcode) { in rvt_ruc_loopback()
2990 if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) { in rvt_ruc_loopback()
2992 wqe->wr.ex.invalidate_rkey)) in rvt_ruc_loopback()
3006 if (wqe->length > qp->r_len) in rvt_ruc_loopback()
3008 switch (wqe->wr.opcode) { in rvt_ruc_loopback()
3011 wqe->wr.ex.invalidate_rkey)) { in rvt_ruc_loopback()
3014 wqe->wr.ex.invalidate_rkey; in rvt_ruc_loopback()
3019 wc.ex.imm_data = wqe->wr.ex.imm_data; in rvt_ruc_loopback()
3030 wc.ex.imm_data = wqe->wr.ex.imm_data; in rvt_ruc_loopback()
3043 if (wqe->length == 0) in rvt_ruc_loopback()
3045 if (unlikely(!rvt_rkey_ok(qp, &qp->r_sge.sge, wqe->length, in rvt_ruc_loopback()
3046 wqe->rdma_wr.remote_addr, in rvt_ruc_loopback()
3047 wqe->rdma_wr.rkey, in rvt_ruc_loopback()
3052 qp->r_sge.total_len = wqe->length; in rvt_ruc_loopback()
3058 if (unlikely(!rvt_rkey_ok(qp, &sqp->s_sge.sge, wqe->length, in rvt_ruc_loopback()
3059 wqe->rdma_wr.remote_addr, in rvt_ruc_loopback()
3060 wqe->rdma_wr.rkey, in rvt_ruc_loopback()
3066 qp->r_sge.sge = wqe->sg_list[0]; in rvt_ruc_loopback()
3067 qp->r_sge.sg_list = wqe->sg_list + 1; in rvt_ruc_loopback()
3068 qp->r_sge.num_sge = wqe->wr.num_sge; in rvt_ruc_loopback()
3069 qp->r_sge.total_len = wqe->length; in rvt_ruc_loopback()
3076 if (unlikely(wqe->atomic_wr.remote_addr & (sizeof(u64) - 1))) in rvt_ruc_loopback()
3079 wqe->atomic_wr.remote_addr, in rvt_ruc_loopback()
3080 wqe->atomic_wr.rkey, in rvt_ruc_loopback()
3085 sdata = wqe->atomic_wr.compare_add; in rvt_ruc_loopback()
3087 (wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ? in rvt_ruc_loopback()
3090 sdata, wqe->atomic_wr.swap); in rvt_ruc_loopback()
3116 if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM) in rvt_ruc_loopback()
3122 wc.byte_len = wqe->length; in rvt_ruc_loopback()
3129 rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED); in rvt_ruc_loopback()
3138 rvt_send_complete(sqp, wqe, send_status); in rvt_ruc_loopback()
3194 rvt_send_complete(sqp, wqe, send_status); in rvt_ruc_loopback()