1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */
3 
4 #include <rdma/ib_umem_odp.h>
5 #include "mlx5_ib.h"
6 #include "umr.h"
7 #include "wr.h"
8 
9 /*
10  * We can't use an array for xlt_emergency_page because dma_map_single doesn't
11  * work on kernel modules memory
12  */
13 void *xlt_emergency_page;
14 static DEFINE_MUTEX(xlt_emergency_page_mutex);
15 
get_umr_enable_mr_mask(void)16 static __be64 get_umr_enable_mr_mask(void)
17 {
18 	u64 result;
19 
20 	result = MLX5_MKEY_MASK_KEY |
21 		 MLX5_MKEY_MASK_FREE;
22 
23 	return cpu_to_be64(result);
24 }
25 
get_umr_disable_mr_mask(void)26 static __be64 get_umr_disable_mr_mask(void)
27 {
28 	u64 result;
29 
30 	result = MLX5_MKEY_MASK_FREE;
31 
32 	return cpu_to_be64(result);
33 }
34 
get_umr_update_translation_mask(void)35 static __be64 get_umr_update_translation_mask(void)
36 {
37 	u64 result;
38 
39 	result = MLX5_MKEY_MASK_LEN |
40 		 MLX5_MKEY_MASK_PAGE_SIZE |
41 		 MLX5_MKEY_MASK_START_ADDR;
42 
43 	return cpu_to_be64(result);
44 }
45 
get_umr_update_access_mask(struct mlx5_ib_dev * dev)46 static __be64 get_umr_update_access_mask(struct mlx5_ib_dev *dev)
47 {
48 	u64 result;
49 
50 	result = MLX5_MKEY_MASK_LR |
51 		 MLX5_MKEY_MASK_LW |
52 		 MLX5_MKEY_MASK_RR |
53 		 MLX5_MKEY_MASK_RW;
54 
55 	if (MLX5_CAP_GEN(dev->mdev, atomic))
56 		result |= MLX5_MKEY_MASK_A;
57 
58 	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
59 		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE;
60 
61 	if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
62 		result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ;
63 
64 	return cpu_to_be64(result);
65 }
66 
get_umr_update_pd_mask(void)67 static __be64 get_umr_update_pd_mask(void)
68 {
69 	u64 result;
70 
71 	result = MLX5_MKEY_MASK_PD;
72 
73 	return cpu_to_be64(result);
74 }
75 
umr_check_mkey_mask(struct mlx5_ib_dev * dev,u64 mask)76 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask)
77 {
78 	if (mask & MLX5_MKEY_MASK_PAGE_SIZE &&
79 	    MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
80 		return -EPERM;
81 
82 	if (mask & MLX5_MKEY_MASK_A &&
83 	    MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled))
84 		return -EPERM;
85 
86 	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE &&
87 	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr))
88 		return -EPERM;
89 
90 	if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ &&
91 	    !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr))
92 		return -EPERM;
93 
94 	return 0;
95 }
96 
97 enum {
98 	MAX_UMR_WR = 128,
99 };
100 
mlx5r_umr_qp_rst2rts(struct mlx5_ib_dev * dev,struct ib_qp * qp)101 static int mlx5r_umr_qp_rst2rts(struct mlx5_ib_dev *dev, struct ib_qp *qp)
102 {
103 	struct ib_qp_attr attr = {};
104 	int ret;
105 
106 	attr.qp_state = IB_QPS_INIT;
107 	attr.port_num = 1;
108 	ret = ib_modify_qp(qp, &attr,
109 			   IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT);
110 	if (ret) {
111 		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
112 		return ret;
113 	}
114 
115 	memset(&attr, 0, sizeof(attr));
116 	attr.qp_state = IB_QPS_RTR;
117 
118 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
119 	if (ret) {
120 		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n");
121 		return ret;
122 	}
123 
124 	memset(&attr, 0, sizeof(attr));
125 	attr.qp_state = IB_QPS_RTS;
126 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
127 	if (ret) {
128 		mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n");
129 		return ret;
130 	}
131 
132 	return 0;
133 }
134 
mlx5r_umr_resource_init(struct mlx5_ib_dev * dev)135 int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev)
136 {
137 	struct ib_qp_init_attr init_attr = {};
138 	struct ib_pd *pd;
139 	struct ib_cq *cq;
140 	struct ib_qp *qp;
141 	int ret;
142 
143 	pd = ib_alloc_pd(&dev->ib_dev, 0);
144 	if (IS_ERR(pd)) {
145 		mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n");
146 		return PTR_ERR(pd);
147 	}
148 
149 	cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ);
150 	if (IS_ERR(cq)) {
151 		mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n");
152 		ret = PTR_ERR(cq);
153 		goto destroy_pd;
154 	}
155 
156 	init_attr.send_cq = cq;
157 	init_attr.recv_cq = cq;
158 	init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
159 	init_attr.cap.max_send_wr = MAX_UMR_WR;
160 	init_attr.cap.max_send_sge = 1;
161 	init_attr.qp_type = MLX5_IB_QPT_REG_UMR;
162 	init_attr.port_num = 1;
163 	qp = ib_create_qp(pd, &init_attr);
164 	if (IS_ERR(qp)) {
165 		mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n");
166 		ret = PTR_ERR(qp);
167 		goto destroy_cq;
168 	}
169 
170 	ret = mlx5r_umr_qp_rst2rts(dev, qp);
171 	if (ret)
172 		goto destroy_qp;
173 
174 	dev->umrc.qp = qp;
175 	dev->umrc.cq = cq;
176 	dev->umrc.pd = pd;
177 
178 	sema_init(&dev->umrc.sem, MAX_UMR_WR);
179 	mutex_init(&dev->umrc.lock);
180 	dev->umrc.state = MLX5_UMR_STATE_ACTIVE;
181 
182 	return 0;
183 
184 destroy_qp:
185 	ib_destroy_qp(qp);
186 destroy_cq:
187 	ib_free_cq(cq);
188 destroy_pd:
189 	ib_dealloc_pd(pd);
190 	return ret;
191 }
192 
mlx5r_umr_resource_cleanup(struct mlx5_ib_dev * dev)193 void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev)
194 {
195 	if (dev->umrc.state == MLX5_UMR_STATE_UNINIT)
196 		return;
197 	ib_destroy_qp(dev->umrc.qp);
198 	ib_free_cq(dev->umrc.cq);
199 	ib_dealloc_pd(dev->umrc.pd);
200 }
201 
mlx5r_umr_recover(struct mlx5_ib_dev * dev)202 static int mlx5r_umr_recover(struct mlx5_ib_dev *dev)
203 {
204 	struct umr_common *umrc = &dev->umrc;
205 	struct ib_qp_attr attr;
206 	int err;
207 
208 	attr.qp_state = IB_QPS_RESET;
209 	err = ib_modify_qp(umrc->qp, &attr, IB_QP_STATE);
210 	if (err) {
211 		mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n");
212 		goto err;
213 	}
214 
215 	err = mlx5r_umr_qp_rst2rts(dev, umrc->qp);
216 	if (err)
217 		goto err;
218 
219 	umrc->state = MLX5_UMR_STATE_ACTIVE;
220 	return 0;
221 
222 err:
223 	umrc->state = MLX5_UMR_STATE_ERR;
224 	return err;
225 }
226 
mlx5r_umr_post_send(struct ib_qp * ibqp,u32 mkey,struct ib_cqe * cqe,struct mlx5r_umr_wqe * wqe,bool with_data)227 static int mlx5r_umr_post_send(struct ib_qp *ibqp, u32 mkey, struct ib_cqe *cqe,
228 			       struct mlx5r_umr_wqe *wqe, bool with_data)
229 {
230 	unsigned int wqe_size =
231 		with_data ? sizeof(struct mlx5r_umr_wqe) :
232 			    sizeof(struct mlx5r_umr_wqe) -
233 				    sizeof(struct mlx5_wqe_data_seg);
234 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
235 	struct mlx5_core_dev *mdev = dev->mdev;
236 	struct mlx5_ib_qp *qp = to_mqp(ibqp);
237 	struct mlx5_wqe_ctrl_seg *ctrl;
238 	union {
239 		struct ib_cqe *ib_cqe;
240 		u64 wr_id;
241 	} id;
242 	void *cur_edge, *seg;
243 	unsigned long flags;
244 	unsigned int idx;
245 	int size, err;
246 
247 	if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR))
248 		return -EIO;
249 
250 	spin_lock_irqsave(&qp->sq.lock, flags);
251 
252 	err = mlx5r_begin_wqe(qp, &seg, &ctrl, &idx, &size, &cur_edge, 0,
253 			      cpu_to_be32(mkey), false, false);
254 	if (WARN_ON(err))
255 		goto out;
256 
257 	qp->sq.wr_data[idx] = MLX5_IB_WR_UMR;
258 
259 	mlx5r_memcpy_send_wqe(&qp->sq, &cur_edge, &seg, &size, wqe, wqe_size);
260 
261 	id.ib_cqe = cqe;
262 	mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, id.wr_id, 0,
263 			 MLX5_FENCE_MODE_INITIATOR_SMALL, MLX5_OPCODE_UMR);
264 
265 	mlx5r_ring_db(qp, 1, ctrl);
266 
267 out:
268 	spin_unlock_irqrestore(&qp->sq.lock, flags);
269 
270 	return err;
271 }
272 
mlx5r_umr_done(struct ib_cq * cq,struct ib_wc * wc)273 static void mlx5r_umr_done(struct ib_cq *cq, struct ib_wc *wc)
274 {
275 	struct mlx5_ib_umr_context *context =
276 		container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe);
277 
278 	context->status = wc->status;
279 	complete(&context->done);
280 }
281 
mlx5r_umr_init_context(struct mlx5r_umr_context * context)282 static inline void mlx5r_umr_init_context(struct mlx5r_umr_context *context)
283 {
284 	context->cqe.done = mlx5r_umr_done;
285 	init_completion(&context->done);
286 }
287 
mlx5r_umr_post_send_wait(struct mlx5_ib_dev * dev,u32 mkey,struct mlx5r_umr_wqe * wqe,bool with_data)288 static int mlx5r_umr_post_send_wait(struct mlx5_ib_dev *dev, u32 mkey,
289 				   struct mlx5r_umr_wqe *wqe, bool with_data)
290 {
291 	struct umr_common *umrc = &dev->umrc;
292 	struct mlx5r_umr_context umr_context;
293 	int err;
294 
295 	err = umr_check_mkey_mask(dev, be64_to_cpu(wqe->ctrl_seg.mkey_mask));
296 	if (WARN_ON(err))
297 		return err;
298 
299 	mlx5r_umr_init_context(&umr_context);
300 
301 	down(&umrc->sem);
302 	while (true) {
303 		mutex_lock(&umrc->lock);
304 		if (umrc->state == MLX5_UMR_STATE_ERR) {
305 			mutex_unlock(&umrc->lock);
306 			err = -EFAULT;
307 			break;
308 		}
309 
310 		if (umrc->state == MLX5_UMR_STATE_RECOVER) {
311 			mutex_unlock(&umrc->lock);
312 			usleep_range(3000, 5000);
313 			continue;
314 		}
315 
316 		err = mlx5r_umr_post_send(umrc->qp, mkey, &umr_context.cqe, wqe,
317 					  with_data);
318 		mutex_unlock(&umrc->lock);
319 		if (err) {
320 			mlx5_ib_warn(dev, "UMR post send failed, err %d\n",
321 				     err);
322 			break;
323 		}
324 
325 		wait_for_completion(&umr_context.done);
326 
327 		if (umr_context.status == IB_WC_SUCCESS)
328 			break;
329 
330 		if (umr_context.status == IB_WC_WR_FLUSH_ERR)
331 			continue;
332 
333 		WARN_ON_ONCE(1);
334 		mlx5_ib_warn(dev,
335 			"reg umr failed (%u). Trying to recover and resubmit the flushed WQEs\n",
336 			umr_context.status);
337 		mutex_lock(&umrc->lock);
338 		err = mlx5r_umr_recover(dev);
339 		mutex_unlock(&umrc->lock);
340 		if (err)
341 			mlx5_ib_warn(dev, "couldn't recover UMR, err %d\n",
342 				     err);
343 		err = -EFAULT;
344 		break;
345 	}
346 	up(&umrc->sem);
347 	return err;
348 }
349 
350 /**
351  * mlx5r_umr_revoke_mr - Fence all DMA on the MR
352  * @mr: The MR to fence
353  *
354  * Upon return the NIC will not be doing any DMA to the pages under the MR,
355  * and any DMA in progress will be completed. Failure of this function
356  * indicates the HW has failed catastrophically.
357  */
mlx5r_umr_revoke_mr(struct mlx5_ib_mr * mr)358 int mlx5r_umr_revoke_mr(struct mlx5_ib_mr *mr)
359 {
360 	struct mlx5_ib_dev *dev = mr_to_mdev(mr);
361 	struct mlx5r_umr_wqe wqe = {};
362 
363 	if (dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
364 		return 0;
365 
366 	wqe.ctrl_seg.mkey_mask |= get_umr_update_pd_mask();
367 	wqe.ctrl_seg.mkey_mask |= get_umr_disable_mr_mask();
368 	wqe.ctrl_seg.flags |= MLX5_UMR_INLINE;
369 
370 	MLX5_SET(mkc, &wqe.mkey_seg, free, 1);
371 	MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(dev->umrc.pd)->pdn);
372 	MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff);
373 	MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0,
374 		 mlx5_mkey_variant(mr->mmkey.key));
375 
376 	return mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false);
377 }
378 
mlx5r_umr_set_access_flags(struct mlx5_ib_dev * dev,struct mlx5_mkey_seg * seg,unsigned int access_flags)379 static void mlx5r_umr_set_access_flags(struct mlx5_ib_dev *dev,
380 				       struct mlx5_mkey_seg *seg,
381 				       unsigned int access_flags)
382 {
383 	bool ro_read = (access_flags & IB_ACCESS_RELAXED_ORDERING) &&
384 		       (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) ||
385 			pcie_relaxed_ordering_enabled(dev->mdev->pdev));
386 
387 	MLX5_SET(mkc, seg, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC));
388 	MLX5_SET(mkc, seg, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
389 	MLX5_SET(mkc, seg, rr, !!(access_flags & IB_ACCESS_REMOTE_READ));
390 	MLX5_SET(mkc, seg, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE));
391 	MLX5_SET(mkc, seg, lr, 1);
392 	MLX5_SET(mkc, seg, relaxed_ordering_write,
393 		 !!(access_flags & IB_ACCESS_RELAXED_ORDERING));
394 	MLX5_SET(mkc, seg, relaxed_ordering_read, ro_read);
395 }
396 
mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr * mr,struct ib_pd * pd,int access_flags)397 int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd,
398 			      int access_flags)
399 {
400 	struct mlx5_ib_dev *dev = mr_to_mdev(mr);
401 	struct mlx5r_umr_wqe wqe = {};
402 	int err;
403 
404 	wqe.ctrl_seg.mkey_mask = get_umr_update_access_mask(dev);
405 	wqe.ctrl_seg.mkey_mask |= get_umr_update_pd_mask();
406 	wqe.ctrl_seg.flags = MLX5_UMR_CHECK_FREE;
407 	wqe.ctrl_seg.flags |= MLX5_UMR_INLINE;
408 
409 	mlx5r_umr_set_access_flags(dev, &wqe.mkey_seg, access_flags);
410 	MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(pd)->pdn);
411 	MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff);
412 	MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0,
413 		 mlx5_mkey_variant(mr->mmkey.key));
414 
415 	err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false);
416 	if (err)
417 		return err;
418 
419 	mr->access_flags = access_flags;
420 	return 0;
421 }
422 
423 #define MLX5_MAX_UMR_CHUNK                                                     \
424 	((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_FLEX_ALIGNMENT)
425 #define MLX5_SPARE_UMR_CHUNK 0x10000
426 
427 /*
428  * Allocate a temporary buffer to hold the per-page information to transfer to
429  * HW. For efficiency this should be as large as it can be, but buffer
430  * allocation failure is not allowed, so try smaller sizes.
431  */
mlx5r_umr_alloc_xlt(size_t * nents,size_t ent_size,gfp_t gfp_mask)432 static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask)
433 {
434 	const size_t xlt_chunk_align = MLX5_UMR_FLEX_ALIGNMENT / ent_size;
435 	size_t size;
436 	void *res = NULL;
437 
438 	static_assert(PAGE_SIZE % MLX5_UMR_FLEX_ALIGNMENT == 0);
439 
440 	/*
441 	 * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the
442 	 * allocation can't trigger any kind of reclaim.
443 	 */
444 	might_sleep();
445 
446 	gfp_mask |= __GFP_ZERO | __GFP_NORETRY;
447 
448 	/*
449 	 * If the system already has a suitable high order page then just use
450 	 * that, but don't try hard to create one. This max is about 1M, so a
451 	 * free x86 huge page will satisfy it.
452 	 */
453 	size = min_t(size_t, ent_size * ALIGN(*nents, xlt_chunk_align),
454 		     MLX5_MAX_UMR_CHUNK);
455 	*nents = size / ent_size;
456 	res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
457 				       get_order(size));
458 	if (res)
459 		return res;
460 
461 	if (size > MLX5_SPARE_UMR_CHUNK) {
462 		size = MLX5_SPARE_UMR_CHUNK;
463 		*nents = size / ent_size;
464 		res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN,
465 					       get_order(size));
466 		if (res)
467 			return res;
468 	}
469 
470 	*nents = PAGE_SIZE / ent_size;
471 	res = (void *)__get_free_page(gfp_mask);
472 	if (res)
473 		return res;
474 
475 	mutex_lock(&xlt_emergency_page_mutex);
476 	memset(xlt_emergency_page, 0, PAGE_SIZE);
477 	return xlt_emergency_page;
478 }
479 
mlx5r_umr_free_xlt(void * xlt,size_t length)480 static void mlx5r_umr_free_xlt(void *xlt, size_t length)
481 {
482 	if (xlt == xlt_emergency_page) {
483 		mutex_unlock(&xlt_emergency_page_mutex);
484 		return;
485 	}
486 
487 	free_pages((unsigned long)xlt, get_order(length));
488 }
489 
mlx5r_umr_unmap_free_xlt(struct mlx5_ib_dev * dev,void * xlt,struct ib_sge * sg)490 static void mlx5r_umr_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt,
491 				     struct ib_sge *sg)
492 {
493 	struct device *ddev = &dev->mdev->pdev->dev;
494 
495 	dma_unmap_single(ddev, sg->addr, sg->length, DMA_TO_DEVICE);
496 	mlx5r_umr_free_xlt(xlt, sg->length);
497 }
498 
499 /*
500  * Create an XLT buffer ready for submission.
501  */
mlx5r_umr_create_xlt(struct mlx5_ib_dev * dev,struct ib_sge * sg,size_t nents,size_t ent_size,unsigned int flags)502 static void *mlx5r_umr_create_xlt(struct mlx5_ib_dev *dev, struct ib_sge *sg,
503 				  size_t nents, size_t ent_size,
504 				  unsigned int flags)
505 {
506 	struct device *ddev = &dev->mdev->pdev->dev;
507 	dma_addr_t dma;
508 	void *xlt;
509 
510 	xlt = mlx5r_umr_alloc_xlt(&nents, ent_size,
511 				 flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC :
512 								  GFP_KERNEL);
513 	sg->length = nents * ent_size;
514 	dma = dma_map_single(ddev, xlt, sg->length, DMA_TO_DEVICE);
515 	if (dma_mapping_error(ddev, dma)) {
516 		mlx5_ib_err(dev, "unable to map DMA during XLT update.\n");
517 		mlx5r_umr_free_xlt(xlt, sg->length);
518 		return NULL;
519 	}
520 	sg->addr = dma;
521 	sg->lkey = dev->umrc.pd->local_dma_lkey;
522 
523 	return xlt;
524 }
525 
526 static void
mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg * ctrl_seg,unsigned int flags,struct ib_sge * sg)527 mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg,
528 				  unsigned int flags, struct ib_sge *sg)
529 {
530 	if (!(flags & MLX5_IB_UPD_XLT_ENABLE))
531 		/* fail if free */
532 		ctrl_seg->flags = MLX5_UMR_CHECK_FREE;
533 	else
534 		/* fail if not free */
535 		ctrl_seg->flags = MLX5_UMR_CHECK_NOT_FREE;
536 	ctrl_seg->xlt_octowords =
537 		cpu_to_be16(mlx5r_umr_get_xlt_octo(sg->length));
538 }
539 
mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev * dev,struct mlx5_mkey_seg * mkey_seg,struct mlx5_ib_mr * mr,unsigned int page_shift)540 static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev,
541 					      struct mlx5_mkey_seg *mkey_seg,
542 					      struct mlx5_ib_mr *mr,
543 					      unsigned int page_shift)
544 {
545 	mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags);
546 	MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn);
547 	MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova);
548 	MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length);
549 	MLX5_SET(mkc, mkey_seg, log_page_size, page_shift);
550 	MLX5_SET(mkc, mkey_seg, qpn, 0xffffff);
551 	MLX5_SET(mkc, mkey_seg, mkey_7_0, mlx5_mkey_variant(mr->mmkey.key));
552 }
553 
554 static void
mlx5r_umr_set_update_xlt_data_seg(struct mlx5_wqe_data_seg * data_seg,struct ib_sge * sg)555 mlx5r_umr_set_update_xlt_data_seg(struct mlx5_wqe_data_seg *data_seg,
556 				  struct ib_sge *sg)
557 {
558 	data_seg->byte_count = cpu_to_be32(sg->length);
559 	data_seg->lkey = cpu_to_be32(sg->lkey);
560 	data_seg->addr = cpu_to_be64(sg->addr);
561 }
562 
mlx5r_umr_update_offset(struct mlx5_wqe_umr_ctrl_seg * ctrl_seg,u64 offset)563 static void mlx5r_umr_update_offset(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg,
564 				    u64 offset)
565 {
566 	u64 octo_offset = mlx5r_umr_get_xlt_octo(offset);
567 
568 	ctrl_seg->xlt_offset = cpu_to_be16(octo_offset & 0xffff);
569 	ctrl_seg->xlt_offset_47_16 = cpu_to_be32(octo_offset >> 16);
570 	ctrl_seg->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN;
571 }
572 
mlx5r_umr_final_update_xlt(struct mlx5_ib_dev * dev,struct mlx5r_umr_wqe * wqe,struct mlx5_ib_mr * mr,struct ib_sge * sg,unsigned int flags)573 static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev,
574 				       struct mlx5r_umr_wqe *wqe,
575 				       struct mlx5_ib_mr *mr, struct ib_sge *sg,
576 				       unsigned int flags)
577 {
578 	bool update_pd_access, update_translation;
579 
580 	if (flags & MLX5_IB_UPD_XLT_ENABLE)
581 		wqe->ctrl_seg.mkey_mask |= get_umr_enable_mr_mask();
582 
583 	update_pd_access = flags & MLX5_IB_UPD_XLT_ENABLE ||
584 			   flags & MLX5_IB_UPD_XLT_PD ||
585 			   flags & MLX5_IB_UPD_XLT_ACCESS;
586 
587 	if (update_pd_access) {
588 		wqe->ctrl_seg.mkey_mask |= get_umr_update_access_mask(dev);
589 		wqe->ctrl_seg.mkey_mask |= get_umr_update_pd_mask();
590 	}
591 
592 	update_translation =
593 		flags & MLX5_IB_UPD_XLT_ENABLE || flags & MLX5_IB_UPD_XLT_ADDR;
594 
595 	if (update_translation) {
596 		wqe->ctrl_seg.mkey_mask |= get_umr_update_translation_mask();
597 		if (!mr->ibmr.length)
598 			MLX5_SET(mkc, &wqe->mkey_seg, length64, 1);
599 	}
600 
601 	wqe->ctrl_seg.xlt_octowords =
602 		cpu_to_be16(mlx5r_umr_get_xlt_octo(sg->length));
603 	wqe->data_seg.byte_count = cpu_to_be32(sg->length);
604 }
605 
606 /*
607  * Send the DMA list to the HW for a normal MR using UMR.
608  * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP
609  * flag may be used.
610  */
mlx5r_umr_update_mr_pas(struct mlx5_ib_mr * mr,unsigned int flags)611 int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags)
612 {
613 	struct mlx5_ib_dev *dev = mr_to_mdev(mr);
614 	struct device *ddev = &dev->mdev->pdev->dev;
615 	struct mlx5r_umr_wqe wqe = {};
616 	struct ib_block_iter biter;
617 	struct mlx5_mtt *cur_mtt;
618 	size_t orig_sg_length;
619 	struct mlx5_mtt *mtt;
620 	size_t final_size;
621 	struct ib_sge sg;
622 	u64 offset = 0;
623 	int err = 0;
624 
625 	if (WARN_ON(mr->umem->is_odp))
626 		return -EINVAL;
627 
628 	mtt = mlx5r_umr_create_xlt(
629 		dev, &sg, ib_umem_num_dma_blocks(mr->umem, 1 << mr->page_shift),
630 		sizeof(*mtt), flags);
631 	if (!mtt)
632 		return -ENOMEM;
633 
634 	orig_sg_length = sg.length;
635 
636 	mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg);
637 	mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr,
638 					  mr->page_shift);
639 	mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg);
640 
641 	cur_mtt = mtt;
642 	rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) {
643 		if (cur_mtt == (void *)mtt + sg.length) {
644 			dma_sync_single_for_device(ddev, sg.addr, sg.length,
645 						   DMA_TO_DEVICE);
646 
647 			err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe,
648 						       true);
649 			if (err)
650 				goto err;
651 			dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
652 						DMA_TO_DEVICE);
653 			offset += sg.length;
654 			mlx5r_umr_update_offset(&wqe.ctrl_seg, offset);
655 
656 			cur_mtt = mtt;
657 		}
658 
659 		cur_mtt->ptag =
660 			cpu_to_be64(rdma_block_iter_dma_address(&biter) |
661 				    MLX5_IB_MTT_PRESENT);
662 
663 		if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP))
664 			cur_mtt->ptag = 0;
665 
666 		cur_mtt++;
667 	}
668 
669 	final_size = (void *)cur_mtt - (void *)mtt;
670 	sg.length = ALIGN(final_size, MLX5_UMR_FLEX_ALIGNMENT);
671 	memset(cur_mtt, 0, sg.length - final_size);
672 	mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
673 
674 	dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE);
675 	err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
676 
677 err:
678 	sg.length = orig_sg_length;
679 	mlx5r_umr_unmap_free_xlt(dev, mtt, &sg);
680 	return err;
681 }
682 
umr_can_use_indirect_mkey(struct mlx5_ib_dev * dev)683 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev)
684 {
685 	return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled);
686 }
687 
mlx5r_umr_update_xlt(struct mlx5_ib_mr * mr,u64 idx,int npages,int page_shift,int flags)688 int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
689 			 int page_shift, int flags)
690 {
691 	int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT)
692 			       ? sizeof(struct mlx5_klm)
693 			       : sizeof(struct mlx5_mtt);
694 	const int page_align = MLX5_UMR_FLEX_ALIGNMENT / desc_size;
695 	struct mlx5_ib_dev *dev = mr_to_mdev(mr);
696 	struct device *ddev = &dev->mdev->pdev->dev;
697 	const int page_mask = page_align - 1;
698 	struct mlx5r_umr_wqe wqe = {};
699 	size_t pages_mapped = 0;
700 	size_t pages_to_map = 0;
701 	size_t size_to_map = 0;
702 	size_t orig_sg_length;
703 	size_t pages_iter;
704 	struct ib_sge sg;
705 	int err = 0;
706 	void *xlt;
707 
708 	if ((flags & MLX5_IB_UPD_XLT_INDIRECT) &&
709 	    !umr_can_use_indirect_mkey(dev))
710 		return -EPERM;
711 
712 	if (WARN_ON(!mr->umem->is_odp))
713 		return -EINVAL;
714 
715 	/* UMR copies MTTs in units of MLX5_UMR_FLEX_ALIGNMENT bytes,
716 	 * so we need to align the offset and length accordingly
717 	 */
718 	if (idx & page_mask) {
719 		npages += idx & page_mask;
720 		idx &= ~page_mask;
721 	}
722 	pages_to_map = ALIGN(npages, page_align);
723 
724 	xlt = mlx5r_umr_create_xlt(dev, &sg, npages, desc_size, flags);
725 	if (!xlt)
726 		return -ENOMEM;
727 
728 	pages_iter = sg.length / desc_size;
729 	orig_sg_length = sg.length;
730 
731 	if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) {
732 		struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
733 		size_t max_pages = ib_umem_odp_num_pages(odp) - idx;
734 
735 		pages_to_map = min_t(size_t, pages_to_map, max_pages);
736 	}
737 
738 	mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg);
739 	mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift);
740 	mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg);
741 
742 	for (pages_mapped = 0;
743 	     pages_mapped < pages_to_map && !err;
744 	     pages_mapped += pages_iter, idx += pages_iter) {
745 		npages = min_t(int, pages_iter, pages_to_map - pages_mapped);
746 		size_to_map = npages * desc_size;
747 		dma_sync_single_for_cpu(ddev, sg.addr, sg.length,
748 					DMA_TO_DEVICE);
749 		mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags);
750 		dma_sync_single_for_device(ddev, sg.addr, sg.length,
751 					   DMA_TO_DEVICE);
752 		sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT);
753 
754 		if (pages_mapped + pages_iter >= pages_to_map)
755 			mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags);
756 		mlx5r_umr_update_offset(&wqe.ctrl_seg, idx * desc_size);
757 		err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true);
758 	}
759 	sg.length = orig_sg_length;
760 	mlx5r_umr_unmap_free_xlt(dev, xlt, &sg);
761 	return err;
762 }
763