1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3 * Copyright (c) 2020 Hewlett Packard Enterprise, Inc. All rights reserved.
4 */
5
6 /*
7 * The rdma_rxe driver supports type 1 or type 2B memory windows.
8 * Type 1 MWs are created by ibv_alloc_mw() verbs calls and bound by
9 * ibv_bind_mw() calls. Type 2 MWs are also created by ibv_alloc_mw()
10 * but bound by bind_mw work requests. The ibv_bind_mw() call is converted
11 * by libibverbs to a bind_mw work request.
12 */
13
14 #include "rxe.h"
15
rxe_alloc_mw(struct ib_mw * ibmw,struct ib_udata * udata)16 int rxe_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata)
17 {
18 struct rxe_mw *mw = to_rmw(ibmw);
19 struct rxe_pd *pd = to_rpd(ibmw->pd);
20 struct rxe_dev *rxe = to_rdev(ibmw->device);
21 int ret;
22
23 rxe_get(pd);
24
25 ret = rxe_add_to_pool(&rxe->mw_pool, mw);
26 if (ret) {
27 rxe_put(pd);
28 return ret;
29 }
30
31 mw->rkey = ibmw->rkey = (mw->elem.index << 8) | rxe_get_next_key(-1);
32 mw->state = (mw->ibmw.type == IB_MW_TYPE_2) ?
33 RXE_MW_STATE_FREE : RXE_MW_STATE_VALID;
34 spin_lock_init(&mw->lock);
35
36 return 0;
37 }
38
rxe_dealloc_mw(struct ib_mw * ibmw)39 int rxe_dealloc_mw(struct ib_mw *ibmw)
40 {
41 struct rxe_mw *mw = to_rmw(ibmw);
42
43 rxe_put(mw);
44
45 return 0;
46 }
47
rxe_check_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr)48 static int rxe_check_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
49 struct rxe_mw *mw, struct rxe_mr *mr)
50 {
51 if (mw->ibmw.type == IB_MW_TYPE_1) {
52 if (unlikely(mw->state != RXE_MW_STATE_VALID)) {
53 pr_err_once(
54 "attempt to bind a type 1 MW not in the valid state\n");
55 return -EINVAL;
56 }
57
58 /* o10-36.2.2 */
59 if (unlikely((mw->access & IB_ZERO_BASED))) {
60 pr_err_once("attempt to bind a zero based type 1 MW\n");
61 return -EINVAL;
62 }
63 }
64
65 if (mw->ibmw.type == IB_MW_TYPE_2) {
66 /* o10-37.2.30 */
67 if (unlikely(mw->state != RXE_MW_STATE_FREE)) {
68 pr_err_once(
69 "attempt to bind a type 2 MW not in the free state\n");
70 return -EINVAL;
71 }
72
73 /* C10-72 */
74 if (unlikely(qp->pd != to_rpd(mw->ibmw.pd))) {
75 pr_err_once(
76 "attempt to bind type 2 MW with qp with different PD\n");
77 return -EINVAL;
78 }
79
80 /* o10-37.2.40 */
81 if (unlikely(!mr || wqe->wr.wr.mw.length == 0)) {
82 pr_err_once(
83 "attempt to invalidate type 2 MW by binding with NULL or zero length MR\n");
84 return -EINVAL;
85 }
86 }
87
88 /* remaining checks only apply to a nonzero MR */
89 if (!mr)
90 return 0;
91
92 if (unlikely(mr->access & IB_ZERO_BASED)) {
93 pr_err_once("attempt to bind MW to zero based MR\n");
94 return -EINVAL;
95 }
96
97 /* C10-73 */
98 if (unlikely(!(mr->access & IB_ACCESS_MW_BIND))) {
99 pr_err_once(
100 "attempt to bind an MW to an MR without bind access\n");
101 return -EINVAL;
102 }
103
104 /* C10-74 */
105 if (unlikely((mw->access &
106 (IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_ATOMIC)) &&
107 !(mr->access & IB_ACCESS_LOCAL_WRITE))) {
108 pr_err_once(
109 "attempt to bind an writeable MW to an MR without local write access\n");
110 return -EINVAL;
111 }
112
113 /* C10-75 */
114 if (mw->access & IB_ZERO_BASED) {
115 if (unlikely(wqe->wr.wr.mw.length > mr->length)) {
116 pr_err_once(
117 "attempt to bind a ZB MW outside of the MR\n");
118 return -EINVAL;
119 }
120 } else {
121 if (unlikely((wqe->wr.wr.mw.addr < mr->iova) ||
122 ((wqe->wr.wr.mw.addr + wqe->wr.wr.mw.length) >
123 (mr->iova + mr->length)))) {
124 pr_err_once(
125 "attempt to bind a VA MW outside of the MR\n");
126 return -EINVAL;
127 }
128 }
129
130 return 0;
131 }
132
rxe_do_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe,struct rxe_mw * mw,struct rxe_mr * mr)133 static void rxe_do_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe,
134 struct rxe_mw *mw, struct rxe_mr *mr)
135 {
136 u32 key = wqe->wr.wr.mw.rkey & 0xff;
137
138 mw->rkey = (mw->rkey & ~0xff) | key;
139 mw->access = wqe->wr.wr.mw.access;
140 mw->state = RXE_MW_STATE_VALID;
141 mw->addr = wqe->wr.wr.mw.addr;
142 mw->length = wqe->wr.wr.mw.length;
143
144 if (mw->mr) {
145 rxe_put(mw->mr);
146 atomic_dec(&mw->mr->num_mw);
147 mw->mr = NULL;
148 }
149
150 if (mw->length) {
151 mw->mr = mr;
152 atomic_inc(&mr->num_mw);
153 rxe_get(mr);
154 }
155
156 if (mw->ibmw.type == IB_MW_TYPE_2) {
157 rxe_get(qp);
158 mw->qp = qp;
159 }
160 }
161
rxe_bind_mw(struct rxe_qp * qp,struct rxe_send_wqe * wqe)162 int rxe_bind_mw(struct rxe_qp *qp, struct rxe_send_wqe *wqe)
163 {
164 int ret;
165 struct rxe_mw *mw;
166 struct rxe_mr *mr;
167 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
168 u32 mw_rkey = wqe->wr.wr.mw.mw_rkey;
169 u32 mr_lkey = wqe->wr.wr.mw.mr_lkey;
170
171 mw = rxe_pool_get_index(&rxe->mw_pool, mw_rkey >> 8);
172 if (unlikely(!mw)) {
173 ret = -EINVAL;
174 goto err;
175 }
176
177 if (unlikely(mw->rkey != mw_rkey)) {
178 ret = -EINVAL;
179 goto err_drop_mw;
180 }
181
182 if (likely(wqe->wr.wr.mw.length)) {
183 mr = rxe_pool_get_index(&rxe->mr_pool, mr_lkey >> 8);
184 if (unlikely(!mr)) {
185 ret = -EINVAL;
186 goto err_drop_mw;
187 }
188
189 if (unlikely(mr->lkey != mr_lkey)) {
190 ret = -EINVAL;
191 goto err_drop_mr;
192 }
193 } else {
194 mr = NULL;
195 }
196
197 spin_lock_bh(&mw->lock);
198
199 ret = rxe_check_bind_mw(qp, wqe, mw, mr);
200 if (ret)
201 goto err_unlock;
202
203 rxe_do_bind_mw(qp, wqe, mw, mr);
204 err_unlock:
205 spin_unlock_bh(&mw->lock);
206 err_drop_mr:
207 if (mr)
208 rxe_put(mr);
209 err_drop_mw:
210 rxe_put(mw);
211 err:
212 return ret;
213 }
214
rxe_check_invalidate_mw(struct rxe_qp * qp,struct rxe_mw * mw)215 static int rxe_check_invalidate_mw(struct rxe_qp *qp, struct rxe_mw *mw)
216 {
217 if (unlikely(mw->state == RXE_MW_STATE_INVALID))
218 return -EINVAL;
219
220 /* o10-37.2.26 */
221 if (unlikely(mw->ibmw.type == IB_MW_TYPE_1))
222 return -EINVAL;
223
224 return 0;
225 }
226
rxe_do_invalidate_mw(struct rxe_mw * mw)227 static void rxe_do_invalidate_mw(struct rxe_mw *mw)
228 {
229 struct rxe_qp *qp;
230 struct rxe_mr *mr;
231
232 /* valid type 2 MW will always have a QP pointer */
233 qp = mw->qp;
234 mw->qp = NULL;
235 rxe_put(qp);
236
237 /* valid type 2 MW will always have an MR pointer */
238 mr = mw->mr;
239 mw->mr = NULL;
240 atomic_dec(&mr->num_mw);
241 rxe_put(mr);
242
243 mw->access = 0;
244 mw->addr = 0;
245 mw->length = 0;
246 mw->state = RXE_MW_STATE_FREE;
247 }
248
rxe_invalidate_mw(struct rxe_qp * qp,u32 rkey)249 int rxe_invalidate_mw(struct rxe_qp *qp, u32 rkey)
250 {
251 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
252 struct rxe_mw *mw;
253 int ret;
254
255 mw = rxe_pool_get_index(&rxe->mw_pool, rkey >> 8);
256 if (!mw) {
257 ret = -EINVAL;
258 goto err;
259 }
260
261 if (rkey != mw->rkey) {
262 ret = -EINVAL;
263 goto err_drop_ref;
264 }
265
266 spin_lock_bh(&mw->lock);
267
268 ret = rxe_check_invalidate_mw(qp, mw);
269 if (ret)
270 goto err_unlock;
271
272 rxe_do_invalidate_mw(mw);
273 err_unlock:
274 spin_unlock_bh(&mw->lock);
275 err_drop_ref:
276 rxe_put(mw);
277 err:
278 return ret;
279 }
280
rxe_lookup_mw(struct rxe_qp * qp,int access,u32 rkey)281 struct rxe_mw *rxe_lookup_mw(struct rxe_qp *qp, int access, u32 rkey)
282 {
283 struct rxe_dev *rxe = to_rdev(qp->ibqp.device);
284 struct rxe_pd *pd = to_rpd(qp->ibqp.pd);
285 struct rxe_mw *mw;
286 int index = rkey >> 8;
287
288 mw = rxe_pool_get_index(&rxe->mw_pool, index);
289 if (!mw)
290 return NULL;
291
292 if (unlikely((mw->rkey != rkey) || rxe_mw_pd(mw) != pd ||
293 (mw->ibmw.type == IB_MW_TYPE_2 && mw->qp != qp) ||
294 (mw->length == 0) ||
295 (access && !(access & mw->access)) ||
296 mw->state != RXE_MW_STATE_VALID)) {
297 rxe_put(mw);
298 return NULL;
299 }
300
301 return mw;
302 }
303
rxe_mw_cleanup(struct rxe_pool_elem * elem)304 void rxe_mw_cleanup(struct rxe_pool_elem *elem)
305 {
306 struct rxe_mw *mw = container_of(elem, typeof(*mw), elem);
307 struct rxe_pd *pd = to_rpd(mw->ibmw.pd);
308
309 rxe_put(pd);
310
311 if (mw->mr) {
312 struct rxe_mr *mr = mw->mr;
313
314 mw->mr = NULL;
315 atomic_dec(&mr->num_mw);
316 rxe_put(mr);
317 }
318
319 if (mw->qp) {
320 struct rxe_qp *qp = mw->qp;
321
322 mw->qp = NULL;
323 rxe_put(qp);
324 }
325
326 mw->access = 0;
327 mw->addr = 0;
328 mw->length = 0;
329 mw->state = RXE_MW_STATE_INVALID;
330 }
331