627 	for (i = 0; i < wqe->wr.num_sge; i++) {  in rvt_swqe_has_lkey()
1783 int rvt_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,  in rvt_post_recv()  argument
1794 		*bad_wr = wr;  in rvt_post_recv()
1798 	for (; wr; wr = wr->next) {  in rvt_post_recv()
1803 		if ((unsigned)wr->num_sge > qp->r_rq.max_sge) {  in rvt_post_recv()
1804 			*bad_wr = wr;  in rvt_post_recv()
1814 			*bad_wr = wr;  in rvt_post_recv()
1823 			wc.wr_id = wr->wr_id;  in rvt_post_recv()
1828 			wqe->wr_id = wr->wr_id;  in rvt_post_recv()
1829 			wqe->num_sge = wr->num_sge;  in rvt_post_recv()
1830 			for (i = 0; i < wr->num_sge; i++) {  in rvt_post_recv()
1831 				wqe->sg_list[i].addr = wr->sg_list[i].addr;  in rvt_post_recv()
1832 				wqe->sg_list[i].length = wr->sg_list[i].length;  in rvt_post_recv()
1833 				wqe->sg_list[i].lkey = wr->sg_list[i].lkey;  in rvt_post_recv()
1867 	const struct ib_send_wr *wr)  in rvt_qp_valid_operation()  argument
1871 	if (wr->opcode >= RVT_OPERATION_MAX || !post_parms[wr->opcode].length)  in rvt_qp_valid_operation()
1873 	if (!(post_parms[wr->opcode].qpt_support & BIT(qp->ibqp.qp_type)))  in rvt_qp_valid_operation()
1875 	if ((post_parms[wr->opcode].flags & RVT_OPERATION_PRIV) &&  in rvt_qp_valid_operation()
1878 	if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC_SGE &&  in rvt_qp_valid_operation()
1879 	    (wr->num_sge == 0 ||  in rvt_qp_valid_operation()
1880 	     wr->sg_list[0].length < sizeof(u64) ||  in rvt_qp_valid_operation()
1881 	     wr->sg_list[0].addr & (sizeof(u64) - 1)))  in rvt_qp_valid_operation()
1883 	if (post_parms[wr->opcode].flags & RVT_OPERATION_ATOMIC &&  in rvt_qp_valid_operation()
1886 	len = post_parms[wr->opcode].length;  in rvt_qp_valid_operation()
1890 		if (qp->ibqp.pd != ud_wr(wr)->ah->pd)  in rvt_qp_valid_operation()
1963 			   const struct ib_send_wr *wr,  in rvt_post_one_wr()  argument
1983 	if (unlikely(wr->num_sge > qp->s_max_sge))  in rvt_post_one_wr()
1986 	ret = rvt_qp_valid_operation(qp, rdi->post_parms, wr);  in rvt_post_one_wr()
2002 	if ((rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL)) {  in rvt_post_one_wr()
2003 		switch (wr->opcode) {  in rvt_post_one_wr()
2006 					      reg_wr(wr)->mr,  in rvt_post_one_wr()
2007 					      reg_wr(wr)->key,  in rvt_post_one_wr()
2008 					      reg_wr(wr)->access);  in rvt_post_one_wr()
2009 			if (ret || !(wr->send_flags & IB_SEND_SIGNALED))  in rvt_post_one_wr()
2013 			if ((wr->send_flags & IB_SEND_FENCE) ||  in rvt_post_one_wr()
2018 					qp, wr->ex.invalidate_rkey);  in rvt_post_one_wr()
2019 				if (ret || !(wr->send_flags & IB_SEND_SIGNALED))  in rvt_post_one_wr()
2028 	reserved_op = rdi->post_parms[wr->opcode].flags &  in rvt_post_one_wr()
2043 	memcpy(&wqe->wr, wr, cplen);  in rvt_post_one_wr()
2047 	if (wr->num_sge) {  in rvt_post_one_wr()
2050 		acc = wr->opcode >= IB_WR_RDMA_READ ?  in rvt_post_one_wr()
2052 		for (i = 0; i < wr->num_sge; i++) {  in rvt_post_one_wr()
2053 			u32 length = wr->sg_list[i].length;  in rvt_post_one_wr()
2058 					  &wr->sg_list[i], acc);  in rvt_post_one_wr()
2066 		wqe->wr.num_sge = j;  in rvt_post_one_wr()
2082 	if (rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL) {  in rvt_post_one_wr()
2086 			wqe->wr.send_flags |= RVT_SEND_COMPLETION_ONLY;  in rvt_post_one_wr()
2106 	if (!(rdi->post_parms[wr->opcode].flags & RVT_OPERATION_LOCAL))  in rvt_post_one_wr()
2110 		wqe->wr.send_flags |= RVT_SEND_RESERVE_USED;  in rvt_post_one_wr()
2113 		wqe->wr.send_flags &= ~RVT_SEND_RESERVE_USED;  in rvt_post_one_wr()
2116 	trace_rvt_post_one_wr(qp, wqe, wr->num_sge);  in rvt_post_one_wr()
2145 int rvt_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,  in rvt_post_send()  argument
2171 	call_send = qp->s_head == READ_ONCE(qp->s_last) && !wr->next;  in rvt_post_send()
2173 	for (; wr; wr = wr->next) {  in rvt_post_send()
2174 		err = rvt_post_one_wr(qp, wr, &call_send);  in rvt_post_send()
2176 			*bad_wr = wr;  in rvt_post_send()
2206 int rvt_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,  in rvt_post_srq_recv()  argument
2213 	for (; wr; wr = wr->next) {  in rvt_post_srq_recv()
2218 		if ((unsigned)wr->num_sge > srq->rq.max_sge) {  in rvt_post_srq_recv()
2219 			*bad_wr = wr;  in rvt_post_srq_recv()
2230 			*bad_wr = wr;  in rvt_post_srq_recv()
2235 		wqe->wr_id = wr->wr_id;  in rvt_post_srq_recv()
2236 		wqe->num_sge = wr->num_sge;  in rvt_post_srq_recv()
2237 		for (i = 0; i < wr->num_sge; i++) {  in rvt_post_srq_recv()
2238 			wqe->sg_list[i].addr = wr->sg_list[i].addr;  in rvt_post_srq_recv()
2239 			wqe->sg_list[i].length = wr->sg_list[i].length;  in rvt_post_srq_recv()
2240 			wqe->sg_list[i].lkey = wr->sg_list[i].lkey;  in rvt_post_srq_recv()
2792 	last = rvt_qp_complete_swqe(qp, wqe, rdi->wc_opcode[wqe->wr.opcode],  in rvt_send_complete()
2983 	sqp->s_sge.num_sge = wqe->wr.num_sge;  in rvt_ruc_loopback()
2985 	switch (wqe->wr.opcode) {  in rvt_ruc_loopback()
2990 		if (!(wqe->wr.send_flags & RVT_SEND_COMPLETION_ONLY)) {  in rvt_ruc_loopback()
2992 						wqe->wr.ex.invalidate_rkey))  in rvt_ruc_loopback()
3008 		switch (wqe->wr.opcode) {  in rvt_ruc_loopback()
3011 						 wqe->wr.ex.invalidate_rkey)) {  in rvt_ruc_loopback()
3014 					wqe->wr.ex.invalidate_rkey;  in rvt_ruc_loopback()
3019 			wc.ex.imm_data = wqe->wr.ex.imm_data;  in rvt_ruc_loopback()
3030 		wc.ex.imm_data = wqe->wr.ex.imm_data;  in rvt_ruc_loopback()
3068 		qp->r_sge.num_sge = wqe->wr.num_sge;  in rvt_ruc_loopback()
3087 			(wqe->wr.opcode == IB_WR_ATOMIC_FETCH_AND_ADD) ?  in rvt_ruc_loopback()
3116 	if (wqe->wr.opcode == IB_WR_RDMA_WRITE_WITH_IMM)  in rvt_ruc_loopback()
3129 	rvt_recv_cq(qp, &wc, wqe->wr.send_flags & IB_SEND_SOLICITED);  in rvt_ruc_loopback()