1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4  */
5 
6 #include <linux/gfp.h>
7 #include <linux/mlx5/qp.h>
8 #include <linux/mlx5/driver.h>
9 #include "mlx5_ib.h"
10 #include "qp.h"
11 
12 static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,
13 			       struct mlx5_core_dct *dct);
14 
15 static struct mlx5_core_rsc_common *
mlx5_get_rsc(struct mlx5_qp_table * table,u32 rsn)16 mlx5_get_rsc(struct mlx5_qp_table *table, u32 rsn)
17 {
18 	struct mlx5_core_rsc_common *common;
19 	unsigned long flags;
20 
21 	spin_lock_irqsave(&table->lock, flags);
22 
23 	common = radix_tree_lookup(&table->tree, rsn);
24 	if (common)
25 		refcount_inc(&common->refcount);
26 
27 	spin_unlock_irqrestore(&table->lock, flags);
28 
29 	return common;
30 }
31 
mlx5_core_put_rsc(struct mlx5_core_rsc_common * common)32 void mlx5_core_put_rsc(struct mlx5_core_rsc_common *common)
33 {
34 	if (refcount_dec_and_test(&common->refcount))
35 		complete(&common->free);
36 }
37 
qp_allowed_event_types(void)38 static u64 qp_allowed_event_types(void)
39 {
40 	u64 mask;
41 
42 	mask = BIT(MLX5_EVENT_TYPE_PATH_MIG) |
43 	       BIT(MLX5_EVENT_TYPE_COMM_EST) |
44 	       BIT(MLX5_EVENT_TYPE_SQ_DRAINED) |
45 	       BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
46 	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR) |
47 	       BIT(MLX5_EVENT_TYPE_PATH_MIG_FAILED) |
48 	       BIT(MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) |
49 	       BIT(MLX5_EVENT_TYPE_WQ_ACCESS_ERROR);
50 
51 	return mask;
52 }
53 
rq_allowed_event_types(void)54 static u64 rq_allowed_event_types(void)
55 {
56 	u64 mask;
57 
58 	mask = BIT(MLX5_EVENT_TYPE_SRQ_LAST_WQE) |
59 	       BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
60 
61 	return mask;
62 }
63 
sq_allowed_event_types(void)64 static u64 sq_allowed_event_types(void)
65 {
66 	return BIT(MLX5_EVENT_TYPE_WQ_CATAS_ERROR);
67 }
68 
dct_allowed_event_types(void)69 static u64 dct_allowed_event_types(void)
70 {
71 	return BIT(MLX5_EVENT_TYPE_DCT_DRAINED);
72 }
73 
is_event_type_allowed(int rsc_type,int event_type)74 static bool is_event_type_allowed(int rsc_type, int event_type)
75 {
76 	switch (rsc_type) {
77 	case MLX5_EVENT_QUEUE_TYPE_QP:
78 		return BIT(event_type) & qp_allowed_event_types();
79 	case MLX5_EVENT_QUEUE_TYPE_RQ:
80 		return BIT(event_type) & rq_allowed_event_types();
81 	case MLX5_EVENT_QUEUE_TYPE_SQ:
82 		return BIT(event_type) & sq_allowed_event_types();
83 	case MLX5_EVENT_QUEUE_TYPE_DCT:
84 		return BIT(event_type) & dct_allowed_event_types();
85 	default:
86 		WARN(1, "Event arrived for unknown resource type");
87 		return false;
88 	}
89 }
90 
dct_event_notifier(struct mlx5_ib_dev * dev,struct mlx5_eqe * eqe)91 static int dct_event_notifier(struct mlx5_ib_dev *dev, struct mlx5_eqe *eqe)
92 {
93 	struct mlx5_core_dct *dct;
94 	unsigned long flags;
95 	u32 qpn;
96 
97 	qpn = be32_to_cpu(eqe->data.dct.dctn) & 0xFFFFFF;
98 	xa_lock_irqsave(&dev->qp_table.dct_xa, flags);
99 	dct = xa_load(&dev->qp_table.dct_xa, qpn);
100 	if (dct)
101 		complete(&dct->drained);
102 	xa_unlock_irqrestore(&dev->qp_table.dct_xa, flags);
103 	return NOTIFY_OK;
104 }
105 
rsc_event_notifier(struct notifier_block * nb,unsigned long type,void * data)106 static int rsc_event_notifier(struct notifier_block *nb,
107 			      unsigned long type, void *data)
108 {
109 	struct mlx5_ib_dev *dev =
110 		container_of(nb, struct mlx5_ib_dev, qp_table.nb);
111 	struct mlx5_core_rsc_common *common;
112 	struct mlx5_eqe *eqe = data;
113 	u8 event_type = (u8)type;
114 	struct mlx5_core_qp *qp;
115 	u32 rsn;
116 
117 	switch (event_type) {
118 	case MLX5_EVENT_TYPE_DCT_DRAINED:
119 		return dct_event_notifier(dev, eqe);
120 	case MLX5_EVENT_TYPE_PATH_MIG:
121 	case MLX5_EVENT_TYPE_COMM_EST:
122 	case MLX5_EVENT_TYPE_SQ_DRAINED:
123 	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
124 	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
125 	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
126 	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
127 	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
128 		rsn = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff;
129 		rsn |= (eqe->data.qp_srq.type << MLX5_USER_INDEX_LEN);
130 		break;
131 	default:
132 		return NOTIFY_DONE;
133 	}
134 
135 	common = mlx5_get_rsc(&dev->qp_table, rsn);
136 	if (!common)
137 		return NOTIFY_OK;
138 
139 	if (!is_event_type_allowed((rsn >> MLX5_USER_INDEX_LEN), event_type))
140 		goto out;
141 
142 	switch (common->res) {
143 	case MLX5_RES_QP:
144 	case MLX5_RES_RQ:
145 	case MLX5_RES_SQ:
146 		qp = (struct mlx5_core_qp *)common;
147 		qp->event(qp, event_type);
148 		/* Need to put resource in event handler */
149 		return NOTIFY_OK;
150 	default:
151 		break;
152 	}
153 out:
154 	mlx5_core_put_rsc(common);
155 
156 	return NOTIFY_OK;
157 }
158 
create_resource_common(struct mlx5_ib_dev * dev,struct mlx5_core_qp * qp,int rsc_type)159 static int create_resource_common(struct mlx5_ib_dev *dev,
160 				  struct mlx5_core_qp *qp, int rsc_type)
161 {
162 	struct mlx5_qp_table *table = &dev->qp_table;
163 	int err;
164 
165 	qp->common.res = rsc_type;
166 	spin_lock_irq(&table->lock);
167 	err = radix_tree_insert(&table->tree,
168 				qp->qpn | (rsc_type << MLX5_USER_INDEX_LEN),
169 				qp);
170 	spin_unlock_irq(&table->lock);
171 	if (err)
172 		return err;
173 
174 	refcount_set(&qp->common.refcount, 1);
175 	init_completion(&qp->common.free);
176 	qp->pid = current->pid;
177 
178 	return 0;
179 }
180 
destroy_resource_common(struct mlx5_ib_dev * dev,struct mlx5_core_qp * qp)181 static void destroy_resource_common(struct mlx5_ib_dev *dev,
182 				    struct mlx5_core_qp *qp)
183 {
184 	struct mlx5_qp_table *table = &dev->qp_table;
185 	unsigned long flags;
186 
187 	spin_lock_irqsave(&table->lock, flags);
188 	radix_tree_delete(&table->tree,
189 			  qp->qpn | (qp->common.res << MLX5_USER_INDEX_LEN));
190 	spin_unlock_irqrestore(&table->lock, flags);
191 	mlx5_core_put_rsc((struct mlx5_core_rsc_common *)qp);
192 	wait_for_completion(&qp->common.free);
193 }
194 
_mlx5_core_destroy_dct(struct mlx5_ib_dev * dev,struct mlx5_core_dct * dct)195 static int _mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,
196 				  struct mlx5_core_dct *dct)
197 {
198 	u32 in[MLX5_ST_SZ_DW(destroy_dct_in)] = {};
199 	struct mlx5_core_qp *qp = &dct->mqp;
200 
201 	MLX5_SET(destroy_dct_in, in, opcode, MLX5_CMD_OP_DESTROY_DCT);
202 	MLX5_SET(destroy_dct_in, in, dctn, qp->qpn);
203 	MLX5_SET(destroy_dct_in, in, uid, qp->uid);
204 	return mlx5_cmd_exec_in(dev->mdev, destroy_dct, in);
205 }
206 
mlx5_core_create_dct(struct mlx5_ib_dev * dev,struct mlx5_core_dct * dct,u32 * in,int inlen,u32 * out,int outlen)207 int mlx5_core_create_dct(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
208 			 u32 *in, int inlen, u32 *out, int outlen)
209 {
210 	struct mlx5_core_qp *qp = &dct->mqp;
211 	int err;
212 
213 	init_completion(&dct->drained);
214 	MLX5_SET(create_dct_in, in, opcode, MLX5_CMD_OP_CREATE_DCT);
215 
216 	err = mlx5_cmd_do(dev->mdev, in, inlen, out, outlen);
217 	if (err)
218 		return err;
219 
220 	qp->qpn = MLX5_GET(create_dct_out, out, dctn);
221 	qp->uid = MLX5_GET(create_dct_in, in, uid);
222 	err = xa_err(xa_store_irq(&dev->qp_table.dct_xa, qp->qpn, dct, GFP_KERNEL));
223 	if (err)
224 		goto err_cmd;
225 
226 	return 0;
227 err_cmd:
228 	_mlx5_core_destroy_dct(dev, dct);
229 	return err;
230 }
231 
mlx5_qpc_create_qp(struct mlx5_ib_dev * dev,struct mlx5_core_qp * qp,u32 * in,int inlen,u32 * out)232 int mlx5_qpc_create_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
233 		       u32 *in, int inlen, u32 *out)
234 {
235 	u32 din[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
236 	int err;
237 
238 	MLX5_SET(create_qp_in, in, opcode, MLX5_CMD_OP_CREATE_QP);
239 
240 	err = mlx5_cmd_exec(dev->mdev, in, inlen, out,
241 			    MLX5_ST_SZ_BYTES(create_qp_out));
242 	if (err)
243 		return err;
244 
245 	qp->uid = MLX5_GET(create_qp_in, in, uid);
246 	qp->qpn = MLX5_GET(create_qp_out, out, qpn);
247 
248 	err = create_resource_common(dev, qp, MLX5_RES_QP);
249 	if (err)
250 		goto err_cmd;
251 
252 	mlx5_debug_qp_add(dev->mdev, qp);
253 
254 	return 0;
255 
256 err_cmd:
257 	MLX5_SET(destroy_qp_in, din, opcode, MLX5_CMD_OP_DESTROY_QP);
258 	MLX5_SET(destroy_qp_in, din, qpn, qp->qpn);
259 	MLX5_SET(destroy_qp_in, din, uid, qp->uid);
260 	mlx5_cmd_exec_in(dev->mdev, destroy_qp, din);
261 	return err;
262 }
263 
mlx5_core_drain_dct(struct mlx5_ib_dev * dev,struct mlx5_core_dct * dct)264 static int mlx5_core_drain_dct(struct mlx5_ib_dev *dev,
265 			       struct mlx5_core_dct *dct)
266 {
267 	u32 in[MLX5_ST_SZ_DW(drain_dct_in)] = {};
268 	struct mlx5_core_qp *qp = &dct->mqp;
269 
270 	MLX5_SET(drain_dct_in, in, opcode, MLX5_CMD_OP_DRAIN_DCT);
271 	MLX5_SET(drain_dct_in, in, dctn, qp->qpn);
272 	MLX5_SET(drain_dct_in, in, uid, qp->uid);
273 	return mlx5_cmd_exec_in(dev->mdev, drain_dct, in);
274 }
275 
mlx5_core_destroy_dct(struct mlx5_ib_dev * dev,struct mlx5_core_dct * dct)276 int mlx5_core_destroy_dct(struct mlx5_ib_dev *dev,
277 			  struct mlx5_core_dct *dct)
278 {
279 	struct mlx5_qp_table *table = &dev->qp_table;
280 	struct mlx5_core_dct *tmp;
281 	int err;
282 
283 	err = mlx5_core_drain_dct(dev, dct);
284 	if (err) {
285 		if (dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)
286 			goto destroy;
287 
288 		return err;
289 	}
290 	wait_for_completion(&dct->drained);
291 
292 destroy:
293 	tmp = xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, dct, XA_ZERO_ENTRY, GFP_KERNEL);
294 	if (WARN_ON(tmp != dct))
295 		return xa_err(tmp) ?: -EINVAL;
296 
297 	err = _mlx5_core_destroy_dct(dev, dct);
298 	if (err) {
299 		xa_cmpxchg_irq(&table->dct_xa, dct->mqp.qpn, XA_ZERO_ENTRY, dct, 0);
300 		return err;
301 	}
302 	xa_erase_irq(&table->dct_xa, dct->mqp.qpn);
303 	return 0;
304 }
305 
mlx5_core_destroy_qp(struct mlx5_ib_dev * dev,struct mlx5_core_qp * qp)306 int mlx5_core_destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp)
307 {
308 	u32 in[MLX5_ST_SZ_DW(destroy_qp_in)] = {};
309 
310 	mlx5_debug_qp_remove(dev->mdev, qp);
311 
312 	destroy_resource_common(dev, qp);
313 
314 	MLX5_SET(destroy_qp_in, in, opcode, MLX5_CMD_OP_DESTROY_QP);
315 	MLX5_SET(destroy_qp_in, in, qpn, qp->qpn);
316 	MLX5_SET(destroy_qp_in, in, uid, qp->uid);
317 	return mlx5_cmd_exec_in(dev->mdev, destroy_qp, in);
318 }
319 
mlx5_core_set_delay_drop(struct mlx5_ib_dev * dev,u32 timeout_usec)320 int mlx5_core_set_delay_drop(struct mlx5_ib_dev *dev,
321 			     u32 timeout_usec)
322 {
323 	u32 in[MLX5_ST_SZ_DW(set_delay_drop_params_in)] = {};
324 
325 	MLX5_SET(set_delay_drop_params_in, in, opcode,
326 		 MLX5_CMD_OP_SET_DELAY_DROP_PARAMS);
327 	MLX5_SET(set_delay_drop_params_in, in, delay_drop_timeout,
328 		 timeout_usec / 100);
329 	return mlx5_cmd_exec_in(dev->mdev, set_delay_drop_params, in);
330 }
331 
332 struct mbox_info {
333 	u32 *in;
334 	u32 *out;
335 	int inlen;
336 	int outlen;
337 };
338 
mbox_alloc(struct mbox_info * mbox,int inlen,int outlen)339 static int mbox_alloc(struct mbox_info *mbox, int inlen, int outlen)
340 {
341 	mbox->inlen  = inlen;
342 	mbox->outlen = outlen;
343 	mbox->in = kzalloc(mbox->inlen, GFP_KERNEL);
344 	mbox->out = kzalloc(mbox->outlen, GFP_KERNEL);
345 	if (!mbox->in || !mbox->out) {
346 		kfree(mbox->in);
347 		kfree(mbox->out);
348 		return -ENOMEM;
349 	}
350 
351 	return 0;
352 }
353 
mbox_free(struct mbox_info * mbox)354 static void mbox_free(struct mbox_info *mbox)
355 {
356 	kfree(mbox->in);
357 	kfree(mbox->out);
358 }
359 
get_ece_from_mbox(void * out,u16 opcode)360 static int get_ece_from_mbox(void *out, u16 opcode)
361 {
362 	int ece = 0;
363 
364 	switch (opcode) {
365 	case MLX5_CMD_OP_INIT2INIT_QP:
366 		ece = MLX5_GET(init2init_qp_out, out, ece);
367 		break;
368 	case MLX5_CMD_OP_INIT2RTR_QP:
369 		ece = MLX5_GET(init2rtr_qp_out, out, ece);
370 		break;
371 	case MLX5_CMD_OP_RTR2RTS_QP:
372 		ece = MLX5_GET(rtr2rts_qp_out, out, ece);
373 		break;
374 	case MLX5_CMD_OP_RTS2RTS_QP:
375 		ece = MLX5_GET(rts2rts_qp_out, out, ece);
376 		break;
377 	case MLX5_CMD_OP_RST2INIT_QP:
378 		ece = MLX5_GET(rst2init_qp_out, out, ece);
379 		break;
380 	default:
381 		break;
382 	}
383 
384 	return ece;
385 }
386 
modify_qp_mbox_alloc(struct mlx5_core_dev * dev,u16 opcode,int qpn,u32 opt_param_mask,void * qpc,struct mbox_info * mbox,u16 uid,u32 ece)387 static int modify_qp_mbox_alloc(struct mlx5_core_dev *dev, u16 opcode, int qpn,
388 				u32 opt_param_mask, void *qpc,
389 				struct mbox_info *mbox, u16 uid, u32 ece)
390 {
391 	mbox->out = NULL;
392 	mbox->in = NULL;
393 
394 #define MBOX_ALLOC(mbox, typ)  \
395 	mbox_alloc(mbox, MLX5_ST_SZ_BYTES(typ##_in), MLX5_ST_SZ_BYTES(typ##_out))
396 
397 #define MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid)                            \
398 	do {                                                                   \
399 		MLX5_SET(typ##_in, in, opcode, _opcode);                       \
400 		MLX5_SET(typ##_in, in, qpn, _qpn);                             \
401 		MLX5_SET(typ##_in, in, uid, _uid);                             \
402 	} while (0)
403 
404 #define MOD_QP_IN_SET_QPC(typ, in, _opcode, _qpn, _opt_p, _qpc, _uid)          \
405 	do {                                                                   \
406 		MOD_QP_IN_SET(typ, in, _opcode, _qpn, _uid);                   \
407 		MLX5_SET(typ##_in, in, opt_param_mask, _opt_p);                \
408 		memcpy(MLX5_ADDR_OF(typ##_in, in, qpc), _qpc,                  \
409 		       MLX5_ST_SZ_BYTES(qpc));                                 \
410 	} while (0)
411 
412 	switch (opcode) {
413 	/* 2RST & 2ERR */
414 	case MLX5_CMD_OP_2RST_QP:
415 		if (MBOX_ALLOC(mbox, qp_2rst))
416 			return -ENOMEM;
417 		MOD_QP_IN_SET(qp_2rst, mbox->in, opcode, qpn, uid);
418 		break;
419 	case MLX5_CMD_OP_2ERR_QP:
420 		if (MBOX_ALLOC(mbox, qp_2err))
421 			return -ENOMEM;
422 		MOD_QP_IN_SET(qp_2err, mbox->in, opcode, qpn, uid);
423 		break;
424 
425 	/* MODIFY with QPC */
426 	case MLX5_CMD_OP_RST2INIT_QP:
427 		if (MBOX_ALLOC(mbox, rst2init_qp))
428 			return -ENOMEM;
429 		MOD_QP_IN_SET_QPC(rst2init_qp, mbox->in, opcode, qpn,
430 				  opt_param_mask, qpc, uid);
431 		MLX5_SET(rst2init_qp_in, mbox->in, ece, ece);
432 		break;
433 	case MLX5_CMD_OP_INIT2RTR_QP:
434 		if (MBOX_ALLOC(mbox, init2rtr_qp))
435 			return -ENOMEM;
436 		MOD_QP_IN_SET_QPC(init2rtr_qp, mbox->in, opcode, qpn,
437 				  opt_param_mask, qpc, uid);
438 		MLX5_SET(init2rtr_qp_in, mbox->in, ece, ece);
439 		break;
440 	case MLX5_CMD_OP_RTR2RTS_QP:
441 		if (MBOX_ALLOC(mbox, rtr2rts_qp))
442 			return -ENOMEM;
443 		MOD_QP_IN_SET_QPC(rtr2rts_qp, mbox->in, opcode, qpn,
444 				  opt_param_mask, qpc, uid);
445 		MLX5_SET(rtr2rts_qp_in, mbox->in, ece, ece);
446 		break;
447 	case MLX5_CMD_OP_RTS2RTS_QP:
448 		if (MBOX_ALLOC(mbox, rts2rts_qp))
449 			return -ENOMEM;
450 		MOD_QP_IN_SET_QPC(rts2rts_qp, mbox->in, opcode, qpn,
451 				  opt_param_mask, qpc, uid);
452 		MLX5_SET(rts2rts_qp_in, mbox->in, ece, ece);
453 		break;
454 	case MLX5_CMD_OP_SQERR2RTS_QP:
455 		if (MBOX_ALLOC(mbox, sqerr2rts_qp))
456 			return -ENOMEM;
457 		MOD_QP_IN_SET_QPC(sqerr2rts_qp, mbox->in, opcode, qpn,
458 				  opt_param_mask, qpc, uid);
459 		break;
460 	case MLX5_CMD_OP_SQD_RTS_QP:
461 		if (MBOX_ALLOC(mbox, sqd2rts_qp))
462 			return -ENOMEM;
463 		MOD_QP_IN_SET_QPC(sqd2rts_qp, mbox->in, opcode, qpn,
464 				  opt_param_mask, qpc, uid);
465 		break;
466 	case MLX5_CMD_OP_INIT2INIT_QP:
467 		if (MBOX_ALLOC(mbox, init2init_qp))
468 			return -ENOMEM;
469 		MOD_QP_IN_SET_QPC(init2init_qp, mbox->in, opcode, qpn,
470 				  opt_param_mask, qpc, uid);
471 		MLX5_SET(init2init_qp_in, mbox->in, ece, ece);
472 		break;
473 	default:
474 		return -EINVAL;
475 	}
476 	return 0;
477 }
478 
mlx5_core_qp_modify(struct mlx5_ib_dev * dev,u16 opcode,u32 opt_param_mask,void * qpc,struct mlx5_core_qp * qp,u32 * ece)479 int mlx5_core_qp_modify(struct mlx5_ib_dev *dev, u16 opcode, u32 opt_param_mask,
480 			void *qpc, struct mlx5_core_qp *qp, u32 *ece)
481 {
482 	struct mbox_info mbox;
483 	int err;
484 
485 	err = modify_qp_mbox_alloc(dev->mdev, opcode, qp->qpn, opt_param_mask,
486 				   qpc, &mbox, qp->uid, (ece) ? *ece : 0);
487 	if (err)
488 		return err;
489 
490 	err = mlx5_cmd_exec(dev->mdev, mbox.in, mbox.inlen, mbox.out,
491 			    mbox.outlen);
492 
493 	if (ece)
494 		*ece = get_ece_from_mbox(mbox.out, opcode);
495 
496 	mbox_free(&mbox);
497 	return err;
498 }
499 
mlx5_init_qp_table(struct mlx5_ib_dev * dev)500 int mlx5_init_qp_table(struct mlx5_ib_dev *dev)
501 {
502 	struct mlx5_qp_table *table = &dev->qp_table;
503 
504 	spin_lock_init(&table->lock);
505 	INIT_RADIX_TREE(&table->tree, GFP_ATOMIC);
506 	xa_init(&table->dct_xa);
507 	mlx5_qp_debugfs_init(dev->mdev);
508 
509 	table->nb.notifier_call = rsc_event_notifier;
510 	mlx5_notifier_register(dev->mdev, &table->nb);
511 
512 	return 0;
513 }
514 
mlx5_cleanup_qp_table(struct mlx5_ib_dev * dev)515 void mlx5_cleanup_qp_table(struct mlx5_ib_dev *dev)
516 {
517 	struct mlx5_qp_table *table = &dev->qp_table;
518 
519 	mlx5_notifier_unregister(dev->mdev, &table->nb);
520 	mlx5_qp_debugfs_cleanup(dev->mdev);
521 }
522 
mlx5_core_qp_query(struct mlx5_ib_dev * dev,struct mlx5_core_qp * qp,u32 * out,int outlen,bool qpc_ext)523 int mlx5_core_qp_query(struct mlx5_ib_dev *dev, struct mlx5_core_qp *qp,
524 		       u32 *out, int outlen, bool qpc_ext)
525 {
526 	u32 in[MLX5_ST_SZ_DW(query_qp_in)] = {};
527 
528 	MLX5_SET(query_qp_in, in, opcode, MLX5_CMD_OP_QUERY_QP);
529 	MLX5_SET(query_qp_in, in, qpn, qp->qpn);
530 	MLX5_SET(query_qp_in, in, qpc_ext, qpc_ext);
531 
532 	return mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, outlen);
533 }
534 
mlx5_core_dct_query(struct mlx5_ib_dev * dev,struct mlx5_core_dct * dct,u32 * out,int outlen)535 int mlx5_core_dct_query(struct mlx5_ib_dev *dev, struct mlx5_core_dct *dct,
536 			u32 *out, int outlen)
537 {
538 	u32 in[MLX5_ST_SZ_DW(query_dct_in)] = {};
539 	struct mlx5_core_qp *qp = &dct->mqp;
540 
541 	MLX5_SET(query_dct_in, in, opcode, MLX5_CMD_OP_QUERY_DCT);
542 	MLX5_SET(query_dct_in, in, dctn, qp->qpn);
543 
544 	return mlx5_cmd_exec(dev->mdev, (void *)&in, sizeof(in), (void *)out,
545 			     outlen);
546 }
547 
mlx5_core_xrcd_alloc(struct mlx5_ib_dev * dev,u32 * xrcdn)548 int mlx5_core_xrcd_alloc(struct mlx5_ib_dev *dev, u32 *xrcdn)
549 {
550 	u32 out[MLX5_ST_SZ_DW(alloc_xrcd_out)] = {};
551 	u32 in[MLX5_ST_SZ_DW(alloc_xrcd_in)] = {};
552 	int err;
553 
554 	MLX5_SET(alloc_xrcd_in, in, opcode, MLX5_CMD_OP_ALLOC_XRCD);
555 	err = mlx5_cmd_exec_inout(dev->mdev, alloc_xrcd, in, out);
556 	if (!err)
557 		*xrcdn = MLX5_GET(alloc_xrcd_out, out, xrcd);
558 	return err;
559 }
560 
mlx5_core_xrcd_dealloc(struct mlx5_ib_dev * dev,u32 xrcdn)561 int mlx5_core_xrcd_dealloc(struct mlx5_ib_dev *dev, u32 xrcdn)
562 {
563 	u32 in[MLX5_ST_SZ_DW(dealloc_xrcd_in)] = {};
564 
565 	MLX5_SET(dealloc_xrcd_in, in, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
566 	MLX5_SET(dealloc_xrcd_in, in, xrcd, xrcdn);
567 	return mlx5_cmd_exec_in(dev->mdev, dealloc_xrcd, in);
568 }
569 
destroy_rq_tracked(struct mlx5_ib_dev * dev,u32 rqn,u16 uid)570 static int destroy_rq_tracked(struct mlx5_ib_dev *dev, u32 rqn, u16 uid)
571 {
572 	u32 in[MLX5_ST_SZ_DW(destroy_rq_in)] = {};
573 
574 	MLX5_SET(destroy_rq_in, in, opcode, MLX5_CMD_OP_DESTROY_RQ);
575 	MLX5_SET(destroy_rq_in, in, rqn, rqn);
576 	MLX5_SET(destroy_rq_in, in, uid, uid);
577 	return mlx5_cmd_exec_in(dev->mdev, destroy_rq, in);
578 }
579 
mlx5_core_create_rq_tracked(struct mlx5_ib_dev * dev,u32 * in,int inlen,struct mlx5_core_qp * rq)580 int mlx5_core_create_rq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,
581 				struct mlx5_core_qp *rq)
582 {
583 	int err;
584 	u32 rqn;
585 
586 	err = mlx5_core_create_rq(dev->mdev, in, inlen, &rqn);
587 	if (err)
588 		return err;
589 
590 	rq->uid = MLX5_GET(create_rq_in, in, uid);
591 	rq->qpn = rqn;
592 	err = create_resource_common(dev, rq, MLX5_RES_RQ);
593 	if (err)
594 		goto err_destroy_rq;
595 
596 	return 0;
597 
598 err_destroy_rq:
599 	destroy_rq_tracked(dev, rq->qpn, rq->uid);
600 
601 	return err;
602 }
603 
mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev * dev,struct mlx5_core_qp * rq)604 int mlx5_core_destroy_rq_tracked(struct mlx5_ib_dev *dev,
605 				 struct mlx5_core_qp *rq)
606 {
607 	destroy_resource_common(dev, rq);
608 	return destroy_rq_tracked(dev, rq->qpn, rq->uid);
609 }
610 
destroy_sq_tracked(struct mlx5_ib_dev * dev,u32 sqn,u16 uid)611 static void destroy_sq_tracked(struct mlx5_ib_dev *dev, u32 sqn, u16 uid)
612 {
613 	u32 in[MLX5_ST_SZ_DW(destroy_sq_in)] = {};
614 
615 	MLX5_SET(destroy_sq_in, in, opcode, MLX5_CMD_OP_DESTROY_SQ);
616 	MLX5_SET(destroy_sq_in, in, sqn, sqn);
617 	MLX5_SET(destroy_sq_in, in, uid, uid);
618 	mlx5_cmd_exec_in(dev->mdev, destroy_sq, in);
619 }
620 
mlx5_core_create_sq_tracked(struct mlx5_ib_dev * dev,u32 * in,int inlen,struct mlx5_core_qp * sq)621 int mlx5_core_create_sq_tracked(struct mlx5_ib_dev *dev, u32 *in, int inlen,
622 				struct mlx5_core_qp *sq)
623 {
624 	u32 out[MLX5_ST_SZ_DW(create_sq_out)] = {};
625 	int err;
626 
627 	MLX5_SET(create_sq_in, in, opcode, MLX5_CMD_OP_CREATE_SQ);
628 	err = mlx5_cmd_exec(dev->mdev, in, inlen, out, sizeof(out));
629 	if (err)
630 		return err;
631 
632 	sq->qpn = MLX5_GET(create_sq_out, out, sqn);
633 	sq->uid = MLX5_GET(create_sq_in, in, uid);
634 	err = create_resource_common(dev, sq, MLX5_RES_SQ);
635 	if (err)
636 		goto err_destroy_sq;
637 
638 	return 0;
639 
640 err_destroy_sq:
641 	destroy_sq_tracked(dev, sq->qpn, sq->uid);
642 
643 	return err;
644 }
645 
mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev * dev,struct mlx5_core_qp * sq)646 void mlx5_core_destroy_sq_tracked(struct mlx5_ib_dev *dev,
647 				  struct mlx5_core_qp *sq)
648 {
649 	destroy_resource_common(dev, sq);
650 	destroy_sq_tracked(dev, sq->qpn, sq->uid);
651 }
652 
mlx5_core_res_hold(struct mlx5_ib_dev * dev,int res_num,enum mlx5_res_type res_type)653 struct mlx5_core_rsc_common *mlx5_core_res_hold(struct mlx5_ib_dev *dev,
654 						int res_num,
655 						enum mlx5_res_type res_type)
656 {
657 	u32 rsn = res_num | (res_type << MLX5_USER_INDEX_LEN);
658 	struct mlx5_qp_table *table = &dev->qp_table;
659 
660 	return mlx5_get_rsc(table, rsn);
661 }
662 
mlx5_core_res_put(struct mlx5_core_rsc_common * res)663 void mlx5_core_res_put(struct mlx5_core_rsc_common *res)
664 {
665 	mlx5_core_put_rsc(res);
666 }
667