1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2021, Mellanox Technologies inc.  All rights reserved.
4  */
5 
6 #include <linux/interrupt.h>
7 #include <linux/notifier.h>
8 #include <linux/mlx5/driver.h>
9 #include <linux/mlx5/vport.h>
10 #include <linux/mlx5/eq.h>
11 #ifdef CONFIG_RFS_ACCEL
12 #include <linux/cpu_rmap.h>
13 #endif
14 #include "mlx5_core.h"
15 #include "lib/eq.h"
16 #include "fpga/core.h"
17 #include "eswitch.h"
18 #include "lib/clock.h"
19 #include "diag/fw_tracer.h"
20 #include "mlx5_irq.h"
21 #include "devlink.h"
22 
23 enum {
24 	MLX5_EQE_OWNER_INIT_VAL	= 0x1,
25 };
26 
27 enum {
28 	MLX5_EQ_STATE_ARMED		= 0x9,
29 	MLX5_EQ_STATE_FIRED		= 0xa,
30 	MLX5_EQ_STATE_ALWAYS_ARMED	= 0xb,
31 };
32 
33 enum {
34 	MLX5_EQ_DOORBEL_OFFSET	= 0x40,
35 };
36 
37 /* budget must be smaller than MLX5_NUM_SPARE_EQE to guarantee that we update
38  * the ci before we polled all the entries in the EQ. MLX5_NUM_SPARE_EQE is
39  * used to set the EQ size, budget must be smaller than the EQ size.
40  */
41 enum {
42 	MLX5_EQ_POLLING_BUDGET	= 128,
43 };
44 
45 static_assert(MLX5_EQ_POLLING_BUDGET <= MLX5_NUM_SPARE_EQE);
46 
47 struct mlx5_eq_table {
48 	struct list_head        comp_eqs_list;
49 	struct mlx5_eq_async    pages_eq;
50 	struct mlx5_eq_async    cmd_eq;
51 	struct mlx5_eq_async    async_eq;
52 
53 	struct atomic_notifier_head nh[MLX5_EVENT_TYPE_MAX];
54 
55 	/* Since CQ DB is stored in async_eq */
56 	struct mlx5_nb          cq_err_nb;
57 
58 	struct mutex            lock; /* sync async eqs creations */
59 	int			num_comp_eqs;
60 	struct mlx5_irq_table	*irq_table;
61 	struct mlx5_irq         **comp_irqs;
62 	struct mlx5_irq         *ctrl_irq;
63 #ifdef CONFIG_RFS_ACCEL
64 	struct cpu_rmap		*rmap;
65 #endif
66 };
67 
68 #define MLX5_ASYNC_EVENT_MASK ((1ull << MLX5_EVENT_TYPE_PATH_MIG)	    | \
69 			       (1ull << MLX5_EVENT_TYPE_COMM_EST)	    | \
70 			       (1ull << MLX5_EVENT_TYPE_SQ_DRAINED)	    | \
71 			       (1ull << MLX5_EVENT_TYPE_CQ_ERROR)	    | \
72 			       (1ull << MLX5_EVENT_TYPE_WQ_CATAS_ERROR)	    | \
73 			       (1ull << MLX5_EVENT_TYPE_PATH_MIG_FAILED)    | \
74 			       (1ull << MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR) | \
75 			       (1ull << MLX5_EVENT_TYPE_WQ_ACCESS_ERROR)    | \
76 			       (1ull << MLX5_EVENT_TYPE_PORT_CHANGE)	    | \
77 			       (1ull << MLX5_EVENT_TYPE_SRQ_CATAS_ERROR)    | \
78 			       (1ull << MLX5_EVENT_TYPE_SRQ_LAST_WQE)	    | \
79 			       (1ull << MLX5_EVENT_TYPE_SRQ_RQ_LIMIT))
80 
mlx5_cmd_destroy_eq(struct mlx5_core_dev * dev,u8 eqn)81 static int mlx5_cmd_destroy_eq(struct mlx5_core_dev *dev, u8 eqn)
82 {
83 	u32 in[MLX5_ST_SZ_DW(destroy_eq_in)] = {};
84 
85 	MLX5_SET(destroy_eq_in, in, opcode, MLX5_CMD_OP_DESTROY_EQ);
86 	MLX5_SET(destroy_eq_in, in, eq_number, eqn);
87 	return mlx5_cmd_exec_in(dev, destroy_eq, in);
88 }
89 
90 /* caller must eventually call mlx5_cq_put on the returned cq */
mlx5_eq_cq_get(struct mlx5_eq * eq,u32 cqn)91 static struct mlx5_core_cq *mlx5_eq_cq_get(struct mlx5_eq *eq, u32 cqn)
92 {
93 	struct mlx5_cq_table *table = &eq->cq_table;
94 	struct mlx5_core_cq *cq = NULL;
95 
96 	rcu_read_lock();
97 	cq = radix_tree_lookup(&table->tree, cqn);
98 	if (likely(cq))
99 		mlx5_cq_hold(cq);
100 	rcu_read_unlock();
101 
102 	return cq;
103 }
104 
mlx5_eq_comp_int(struct notifier_block * nb,__always_unused unsigned long action,__always_unused void * data)105 static int mlx5_eq_comp_int(struct notifier_block *nb,
106 			    __always_unused unsigned long action,
107 			    __always_unused void *data)
108 {
109 	struct mlx5_eq_comp *eq_comp =
110 		container_of(nb, struct mlx5_eq_comp, irq_nb);
111 	struct mlx5_eq *eq = &eq_comp->core;
112 	struct mlx5_eqe *eqe;
113 	int num_eqes = 0;
114 	u32 cqn = -1;
115 
116 	eqe = next_eqe_sw(eq);
117 	if (!eqe)
118 		goto out;
119 
120 	do {
121 		struct mlx5_core_cq *cq;
122 
123 		/* Make sure we read EQ entry contents after we've
124 		 * checked the ownership bit.
125 		 */
126 		dma_rmb();
127 		/* Assume (eqe->type) is always MLX5_EVENT_TYPE_COMP */
128 		cqn = be32_to_cpu(eqe->data.comp.cqn) & 0xffffff;
129 
130 		cq = mlx5_eq_cq_get(eq, cqn);
131 		if (likely(cq)) {
132 			++cq->arm_sn;
133 			cq->comp(cq, eqe);
134 			mlx5_cq_put(cq);
135 		} else {
136 			dev_dbg_ratelimited(eq->dev->device,
137 					    "Completion event for bogus CQ 0x%x\n", cqn);
138 		}
139 
140 		++eq->cons_index;
141 
142 	} while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
143 
144 out:
145 	eq_update_ci(eq, 1);
146 
147 	if (cqn != -1)
148 		tasklet_schedule(&eq_comp->tasklet_ctx.task);
149 
150 	return 0;
151 }
152 
153 /* Some architectures don't latch interrupts when they are disabled, so using
154  * mlx5_eq_poll_irq_disabled could end up losing interrupts while trying to
155  * avoid losing them.  It is not recommended to use it, unless this is the last
156  * resort.
157  */
mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp * eq)158 u32 mlx5_eq_poll_irq_disabled(struct mlx5_eq_comp *eq)
159 {
160 	u32 count_eqe;
161 
162 	disable_irq(eq->core.irqn);
163 	count_eqe = eq->core.cons_index;
164 	mlx5_eq_comp_int(&eq->irq_nb, 0, NULL);
165 	count_eqe = eq->core.cons_index - count_eqe;
166 	enable_irq(eq->core.irqn);
167 
168 	return count_eqe;
169 }
170 
mlx5_eq_async_int_lock(struct mlx5_eq_async * eq,bool recovery,unsigned long * flags)171 static void mlx5_eq_async_int_lock(struct mlx5_eq_async *eq, bool recovery,
172 				   unsigned long *flags)
173 	__acquires(&eq->lock)
174 {
175 	if (!recovery)
176 		spin_lock(&eq->lock);
177 	else
178 		spin_lock_irqsave(&eq->lock, *flags);
179 }
180 
mlx5_eq_async_int_unlock(struct mlx5_eq_async * eq,bool recovery,unsigned long * flags)181 static void mlx5_eq_async_int_unlock(struct mlx5_eq_async *eq, bool recovery,
182 				     unsigned long *flags)
183 	__releases(&eq->lock)
184 {
185 	if (!recovery)
186 		spin_unlock(&eq->lock);
187 	else
188 		spin_unlock_irqrestore(&eq->lock, *flags);
189 }
190 
191 enum async_eq_nb_action {
192 	ASYNC_EQ_IRQ_HANDLER = 0,
193 	ASYNC_EQ_RECOVER = 1,
194 };
195 
mlx5_eq_async_int(struct notifier_block * nb,unsigned long action,void * data)196 static int mlx5_eq_async_int(struct notifier_block *nb,
197 			     unsigned long action, void *data)
198 {
199 	struct mlx5_eq_async *eq_async =
200 		container_of(nb, struct mlx5_eq_async, irq_nb);
201 	struct mlx5_eq *eq = &eq_async->core;
202 	struct mlx5_eq_table *eqt;
203 	struct mlx5_core_dev *dev;
204 	struct mlx5_eqe *eqe;
205 	unsigned long flags;
206 	int num_eqes = 0;
207 	bool recovery;
208 
209 	dev = eq->dev;
210 	eqt = dev->priv.eq_table;
211 
212 	recovery = action == ASYNC_EQ_RECOVER;
213 	mlx5_eq_async_int_lock(eq_async, recovery, &flags);
214 
215 	eqe = next_eqe_sw(eq);
216 	if (!eqe)
217 		goto out;
218 
219 	do {
220 		/*
221 		 * Make sure we read EQ entry contents after we've
222 		 * checked the ownership bit.
223 		 */
224 		dma_rmb();
225 
226 		atomic_notifier_call_chain(&eqt->nh[eqe->type], eqe->type, eqe);
227 		atomic_notifier_call_chain(&eqt->nh[MLX5_EVENT_TYPE_NOTIFY_ANY], eqe->type, eqe);
228 
229 		++eq->cons_index;
230 
231 	} while ((++num_eqes < MLX5_EQ_POLLING_BUDGET) && (eqe = next_eqe_sw(eq)));
232 
233 out:
234 	eq_update_ci(eq, 1);
235 	mlx5_eq_async_int_unlock(eq_async, recovery, &flags);
236 
237 	return unlikely(recovery) ? num_eqes : 0;
238 }
239 
mlx5_cmd_eq_recover(struct mlx5_core_dev * dev)240 void mlx5_cmd_eq_recover(struct mlx5_core_dev *dev)
241 {
242 	struct mlx5_eq_async *eq = &dev->priv.eq_table->cmd_eq;
243 	int eqes;
244 
245 	eqes = mlx5_eq_async_int(&eq->irq_nb, ASYNC_EQ_RECOVER, NULL);
246 	if (eqes)
247 		mlx5_core_warn(dev, "Recovered %d EQEs on cmd_eq\n", eqes);
248 }
249 
init_eq_buf(struct mlx5_eq * eq)250 static void init_eq_buf(struct mlx5_eq *eq)
251 {
252 	struct mlx5_eqe *eqe;
253 	int i;
254 
255 	for (i = 0; i < eq_get_size(eq); i++) {
256 		eqe = get_eqe(eq, i);
257 		eqe->owner = MLX5_EQE_OWNER_INIT_VAL;
258 	}
259 }
260 
261 static int
create_map_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct mlx5_eq_param * param)262 create_map_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
263 	      struct mlx5_eq_param *param)
264 {
265 	u8 log_eq_size = order_base_2(param->nent + MLX5_NUM_SPARE_EQE);
266 	struct mlx5_cq_table *cq_table = &eq->cq_table;
267 	u32 out[MLX5_ST_SZ_DW(create_eq_out)] = {0};
268 	u8 log_eq_stride = ilog2(MLX5_EQE_SIZE);
269 	struct mlx5_priv *priv = &dev->priv;
270 	__be64 *pas;
271 	u16 vecidx;
272 	void *eqc;
273 	int inlen;
274 	u32 *in;
275 	int err;
276 	int i;
277 
278 	/* Init CQ table */
279 	memset(cq_table, 0, sizeof(*cq_table));
280 	spin_lock_init(&cq_table->lock);
281 	INIT_RADIX_TREE(&cq_table->tree, GFP_ATOMIC);
282 
283 	eq->cons_index = 0;
284 
285 	err = mlx5_frag_buf_alloc_node(dev, wq_get_byte_sz(log_eq_size, log_eq_stride),
286 				       &eq->frag_buf, dev->priv.numa_node);
287 	if (err)
288 		return err;
289 
290 	mlx5_init_fbc(eq->frag_buf.frags, log_eq_stride, log_eq_size, &eq->fbc);
291 	init_eq_buf(eq);
292 
293 	eq->irq = param->irq;
294 	vecidx = mlx5_irq_get_index(eq->irq);
295 
296 	inlen = MLX5_ST_SZ_BYTES(create_eq_in) +
297 		MLX5_FLD_SZ_BYTES(create_eq_in, pas[0]) * eq->frag_buf.npages;
298 
299 	in = kvzalloc(inlen, GFP_KERNEL);
300 	if (!in) {
301 		err = -ENOMEM;
302 		goto err_buf;
303 	}
304 
305 	pas = (__be64 *)MLX5_ADDR_OF(create_eq_in, in, pas);
306 	mlx5_fill_page_frag_array(&eq->frag_buf, pas);
307 
308 	MLX5_SET(create_eq_in, in, opcode, MLX5_CMD_OP_CREATE_EQ);
309 	if (!param->mask[0] && MLX5_CAP_GEN(dev, log_max_uctx))
310 		MLX5_SET(create_eq_in, in, uid, MLX5_SHARED_RESOURCE_UID);
311 
312 	for (i = 0; i < 4; i++)
313 		MLX5_ARRAY_SET64(create_eq_in, in, event_bitmask, i,
314 				 param->mask[i]);
315 
316 	eqc = MLX5_ADDR_OF(create_eq_in, in, eq_context_entry);
317 	MLX5_SET(eqc, eqc, log_eq_size, eq->fbc.log_sz);
318 	MLX5_SET(eqc, eqc, uar_page, priv->uar->index);
319 	MLX5_SET(eqc, eqc, intr, vecidx);
320 	MLX5_SET(eqc, eqc, log_page_size,
321 		 eq->frag_buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
322 
323 	err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out));
324 	if (err)
325 		goto err_in;
326 
327 	eq->vecidx = vecidx;
328 	eq->eqn = MLX5_GET(create_eq_out, out, eq_number);
329 	eq->irqn = pci_irq_vector(dev->pdev, vecidx);
330 	eq->dev = dev;
331 	eq->doorbell = priv->uar->map + MLX5_EQ_DOORBEL_OFFSET;
332 
333 	err = mlx5_debug_eq_add(dev, eq);
334 	if (err)
335 		goto err_eq;
336 
337 	kvfree(in);
338 	return 0;
339 
340 err_eq:
341 	mlx5_cmd_destroy_eq(dev, eq->eqn);
342 
343 err_in:
344 	kvfree(in);
345 
346 err_buf:
347 	mlx5_frag_buf_free(dev, &eq->frag_buf);
348 	return err;
349 }
350 
351 /**
352  * mlx5_eq_enable - Enable EQ for receiving EQEs
353  * @dev : Device which owns the eq
354  * @eq  : EQ to enable
355  * @nb  : Notifier call block
356  *
357  * Must be called after EQ is created in device.
358  *
359  * @return: 0 if no error
360  */
mlx5_eq_enable(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct notifier_block * nb)361 int mlx5_eq_enable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
362 		   struct notifier_block *nb)
363 {
364 	int err;
365 
366 	err = mlx5_irq_attach_nb(eq->irq, nb);
367 	if (!err)
368 		eq_update_ci(eq, 1);
369 
370 	return err;
371 }
372 EXPORT_SYMBOL(mlx5_eq_enable);
373 
374 /**
375  * mlx5_eq_disable - Disable EQ for receiving EQEs
376  * @dev : Device which owns the eq
377  * @eq  : EQ to disable
378  * @nb  : Notifier call block
379  *
380  * Must be called before EQ is destroyed.
381  */
mlx5_eq_disable(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct notifier_block * nb)382 void mlx5_eq_disable(struct mlx5_core_dev *dev, struct mlx5_eq *eq,
383 		     struct notifier_block *nb)
384 {
385 	mlx5_irq_detach_nb(eq->irq, nb);
386 }
387 EXPORT_SYMBOL(mlx5_eq_disable);
388 
destroy_unmap_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq)389 static int destroy_unmap_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
390 {
391 	int err;
392 
393 	mlx5_debug_eq_remove(dev, eq);
394 
395 	err = mlx5_cmd_destroy_eq(dev, eq->eqn);
396 	if (err)
397 		mlx5_core_warn(dev, "failed to destroy a previously created eq: eqn %d\n",
398 			       eq->eqn);
399 
400 	mlx5_frag_buf_free(dev, &eq->frag_buf);
401 	return err;
402 }
403 
mlx5_eq_add_cq(struct mlx5_eq * eq,struct mlx5_core_cq * cq)404 int mlx5_eq_add_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
405 {
406 	struct mlx5_cq_table *table = &eq->cq_table;
407 	int err;
408 
409 	spin_lock(&table->lock);
410 	err = radix_tree_insert(&table->tree, cq->cqn, cq);
411 	spin_unlock(&table->lock);
412 
413 	return err;
414 }
415 
mlx5_eq_del_cq(struct mlx5_eq * eq,struct mlx5_core_cq * cq)416 void mlx5_eq_del_cq(struct mlx5_eq *eq, struct mlx5_core_cq *cq)
417 {
418 	struct mlx5_cq_table *table = &eq->cq_table;
419 	struct mlx5_core_cq *tmp;
420 
421 	spin_lock(&table->lock);
422 	tmp = radix_tree_delete(&table->tree, cq->cqn);
423 	spin_unlock(&table->lock);
424 
425 	if (!tmp) {
426 		mlx5_core_dbg(eq->dev, "cq 0x%x not found in eq 0x%x tree\n",
427 			      eq->eqn, cq->cqn);
428 		return;
429 	}
430 
431 	if (tmp != cq)
432 		mlx5_core_dbg(eq->dev, "corruption on cqn 0x%x in eq 0x%x\n",
433 			      eq->eqn, cq->cqn);
434 }
435 
mlx5_eq_table_init(struct mlx5_core_dev * dev)436 int mlx5_eq_table_init(struct mlx5_core_dev *dev)
437 {
438 	struct mlx5_eq_table *eq_table;
439 	int i;
440 
441 	eq_table = kvzalloc_node(sizeof(*eq_table), GFP_KERNEL,
442 				 dev->priv.numa_node);
443 	if (!eq_table)
444 		return -ENOMEM;
445 
446 	dev->priv.eq_table = eq_table;
447 
448 	mlx5_eq_debugfs_init(dev);
449 
450 	mutex_init(&eq_table->lock);
451 	for (i = 0; i < MLX5_EVENT_TYPE_MAX; i++)
452 		ATOMIC_INIT_NOTIFIER_HEAD(&eq_table->nh[i]);
453 
454 	eq_table->irq_table = mlx5_irq_table_get(dev);
455 	return 0;
456 }
457 
mlx5_eq_table_cleanup(struct mlx5_core_dev * dev)458 void mlx5_eq_table_cleanup(struct mlx5_core_dev *dev)
459 {
460 	mlx5_eq_debugfs_cleanup(dev);
461 	kvfree(dev->priv.eq_table);
462 }
463 
464 /* Async EQs */
465 
create_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq,struct mlx5_eq_param * param)466 static int create_async_eq(struct mlx5_core_dev *dev,
467 			   struct mlx5_eq *eq, struct mlx5_eq_param *param)
468 {
469 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
470 	int err;
471 
472 	mutex_lock(&eq_table->lock);
473 	err = create_map_eq(dev, eq, param);
474 	mutex_unlock(&eq_table->lock);
475 	return err;
476 }
477 
destroy_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq * eq)478 static int destroy_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
479 {
480 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
481 	int err;
482 
483 	mutex_lock(&eq_table->lock);
484 	err = destroy_unmap_eq(dev, eq);
485 	mutex_unlock(&eq_table->lock);
486 	return err;
487 }
488 
cq_err_event_notifier(struct notifier_block * nb,unsigned long type,void * data)489 static int cq_err_event_notifier(struct notifier_block *nb,
490 				 unsigned long type, void *data)
491 {
492 	struct mlx5_eq_table *eqt;
493 	struct mlx5_core_cq *cq;
494 	struct mlx5_eqe *eqe;
495 	struct mlx5_eq *eq;
496 	u32 cqn;
497 
498 	/* type == MLX5_EVENT_TYPE_CQ_ERROR */
499 
500 	eqt = mlx5_nb_cof(nb, struct mlx5_eq_table, cq_err_nb);
501 	eq  = &eqt->async_eq.core;
502 	eqe = data;
503 
504 	cqn = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff;
505 	mlx5_core_warn(eq->dev, "CQ error on CQN 0x%x, syndrome 0x%x\n",
506 		       cqn, eqe->data.cq_err.syndrome);
507 
508 	cq = mlx5_eq_cq_get(eq, cqn);
509 	if (unlikely(!cq)) {
510 		mlx5_core_warn(eq->dev, "Async event for bogus CQ 0x%x\n", cqn);
511 		return NOTIFY_OK;
512 	}
513 
514 	if (cq->event)
515 		cq->event(cq, type);
516 
517 	mlx5_cq_put(cq);
518 
519 	return NOTIFY_OK;
520 }
521 
gather_user_async_events(struct mlx5_core_dev * dev,u64 mask[4])522 static void gather_user_async_events(struct mlx5_core_dev *dev, u64 mask[4])
523 {
524 	__be64 *user_unaffiliated_events;
525 	__be64 *user_affiliated_events;
526 	int i;
527 
528 	user_affiliated_events =
529 		MLX5_CAP_DEV_EVENT(dev, user_affiliated_events);
530 	user_unaffiliated_events =
531 		MLX5_CAP_DEV_EVENT(dev, user_unaffiliated_events);
532 
533 	for (i = 0; i < 4; i++)
534 		mask[i] |= be64_to_cpu(user_affiliated_events[i] |
535 				       user_unaffiliated_events[i]);
536 }
537 
gather_async_events_mask(struct mlx5_core_dev * dev,u64 mask[4])538 static void gather_async_events_mask(struct mlx5_core_dev *dev, u64 mask[4])
539 {
540 	u64 async_event_mask = MLX5_ASYNC_EVENT_MASK;
541 
542 	if (MLX5_VPORT_MANAGER(dev))
543 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_NIC_VPORT_CHANGE);
544 
545 	if (MLX5_CAP_GEN(dev, general_notification_event))
546 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_GENERAL_EVENT);
547 
548 	if (MLX5_CAP_GEN(dev, port_module_event))
549 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PORT_MODULE_EVENT);
550 	else
551 		mlx5_core_dbg(dev, "port_module_event is not set\n");
552 
553 	if (MLX5_PPS_CAP(dev))
554 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_PPS_EVENT);
555 
556 	if (MLX5_CAP_GEN(dev, fpga))
557 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_FPGA_ERROR) |
558 				    (1ull << MLX5_EVENT_TYPE_FPGA_QP_ERROR);
559 	if (MLX5_CAP_GEN_MAX(dev, dct))
560 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DCT_DRAINED);
561 
562 	if (MLX5_CAP_GEN(dev, temp_warn_event))
563 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_TEMP_WARN_EVENT);
564 
565 	if (MLX5_CAP_MCAM_REG(dev, tracer_registers))
566 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_DEVICE_TRACER);
567 
568 	if (MLX5_CAP_GEN(dev, max_num_of_monitor_counters))
569 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_MONITOR_COUNTER);
570 
571 	if (mlx5_eswitch_is_funcs_handler(dev))
572 		async_event_mask |=
573 			(1ull << MLX5_EVENT_TYPE_ESW_FUNCTIONS_CHANGED);
574 
575 	if (MLX5_CAP_GEN_MAX(dev, vhca_state))
576 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_VHCA_STATE_CHANGE);
577 
578 	if (MLX5_CAP_MACSEC(dev, log_max_macsec_offload))
579 		async_event_mask |= (1ull << MLX5_EVENT_TYPE_OBJECT_CHANGE);
580 
581 	mask[0] = async_event_mask;
582 
583 	if (MLX5_CAP_GEN(dev, event_cap))
584 		gather_user_async_events(dev, mask);
585 }
586 
587 static int
setup_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq_async * eq,struct mlx5_eq_param * param,const char * name)588 setup_async_eq(struct mlx5_core_dev *dev, struct mlx5_eq_async *eq,
589 	       struct mlx5_eq_param *param, const char *name)
590 {
591 	int err;
592 
593 	eq->irq_nb.notifier_call = mlx5_eq_async_int;
594 	spin_lock_init(&eq->lock);
595 
596 	err = create_async_eq(dev, &eq->core, param);
597 	if (err) {
598 		mlx5_core_warn(dev, "failed to create %s EQ %d\n", name, err);
599 		return err;
600 	}
601 	err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
602 	if (err) {
603 		mlx5_core_warn(dev, "failed to enable %s EQ %d\n", name, err);
604 		destroy_async_eq(dev, &eq->core);
605 	}
606 	return err;
607 }
608 
cleanup_async_eq(struct mlx5_core_dev * dev,struct mlx5_eq_async * eq,const char * name)609 static void cleanup_async_eq(struct mlx5_core_dev *dev,
610 			     struct mlx5_eq_async *eq, const char *name)
611 {
612 	int err;
613 
614 	mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
615 	err = destroy_async_eq(dev, &eq->core);
616 	if (err)
617 		mlx5_core_err(dev, "failed to destroy %s eq, err(%d)\n",
618 			      name, err);
619 }
620 
async_eq_depth_devlink_param_get(struct mlx5_core_dev * dev)621 static u16 async_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
622 {
623 	struct devlink *devlink = priv_to_devlink(dev);
624 	union devlink_param_value val;
625 	int err;
626 
627 	err = devlink_param_driverinit_value_get(devlink,
628 						 DEVLINK_PARAM_GENERIC_ID_EVENT_EQ_SIZE,
629 						 &val);
630 	if (!err)
631 		return val.vu32;
632 	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
633 	return MLX5_NUM_ASYNC_EQE;
634 }
create_async_eqs(struct mlx5_core_dev * dev)635 static int create_async_eqs(struct mlx5_core_dev *dev)
636 {
637 	struct mlx5_eq_table *table = dev->priv.eq_table;
638 	struct mlx5_eq_param param = {};
639 	int err;
640 
641 	/* All the async_eqs are using single IRQ, request one IRQ and share its
642 	 * index among all the async_eqs of this device.
643 	 */
644 	table->ctrl_irq = mlx5_ctrl_irq_request(dev);
645 	if (IS_ERR(table->ctrl_irq))
646 		return PTR_ERR(table->ctrl_irq);
647 
648 	MLX5_NB_INIT(&table->cq_err_nb, cq_err_event_notifier, CQ_ERROR);
649 	mlx5_eq_notifier_register(dev, &table->cq_err_nb);
650 
651 	param = (struct mlx5_eq_param) {
652 		.irq = table->ctrl_irq,
653 		.nent = MLX5_NUM_CMD_EQE,
654 		.mask[0] = 1ull << MLX5_EVENT_TYPE_CMD,
655 	};
656 	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_CREATE_EQ);
657 	err = setup_async_eq(dev, &table->cmd_eq, &param, "cmd");
658 	if (err)
659 		goto err1;
660 
661 	mlx5_cmd_use_events(dev);
662 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
663 
664 	param = (struct mlx5_eq_param) {
665 		.irq = table->ctrl_irq,
666 		.nent = async_eq_depth_devlink_param_get(dev),
667 	};
668 
669 	gather_async_events_mask(dev, param.mask);
670 	err = setup_async_eq(dev, &table->async_eq, &param, "async");
671 	if (err)
672 		goto err2;
673 
674 	param = (struct mlx5_eq_param) {
675 		.irq = table->ctrl_irq,
676 		.nent = /* TODO: sriov max_vf + */ 1,
677 		.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_REQUEST,
678 	};
679 
680 	err = setup_async_eq(dev, &table->pages_eq, &param, "pages");
681 	if (err)
682 		goto err3;
683 
684 	return 0;
685 
686 err3:
687 	cleanup_async_eq(dev, &table->async_eq, "async");
688 err2:
689 	mlx5_cmd_use_polling(dev);
690 	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
691 err1:
692 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
693 	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
694 	mlx5_ctrl_irq_release(table->ctrl_irq);
695 	return err;
696 }
697 
destroy_async_eqs(struct mlx5_core_dev * dev)698 static void destroy_async_eqs(struct mlx5_core_dev *dev)
699 {
700 	struct mlx5_eq_table *table = dev->priv.eq_table;
701 
702 	cleanup_async_eq(dev, &table->pages_eq, "pages");
703 	cleanup_async_eq(dev, &table->async_eq, "async");
704 	mlx5_cmd_allowed_opcode(dev, MLX5_CMD_OP_DESTROY_EQ);
705 	mlx5_cmd_use_polling(dev);
706 	cleanup_async_eq(dev, &table->cmd_eq, "cmd");
707 	mlx5_cmd_allowed_opcode(dev, CMD_ALLOWED_OPCODE_ALL);
708 	mlx5_eq_notifier_unregister(dev, &table->cq_err_nb);
709 	mlx5_ctrl_irq_release(table->ctrl_irq);
710 }
711 
mlx5_get_async_eq(struct mlx5_core_dev * dev)712 struct mlx5_eq *mlx5_get_async_eq(struct mlx5_core_dev *dev)
713 {
714 	return &dev->priv.eq_table->async_eq.core;
715 }
716 
mlx5_eq_synchronize_async_irq(struct mlx5_core_dev * dev)717 void mlx5_eq_synchronize_async_irq(struct mlx5_core_dev *dev)
718 {
719 	synchronize_irq(dev->priv.eq_table->async_eq.core.irqn);
720 }
721 
mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev * dev)722 void mlx5_eq_synchronize_cmd_irq(struct mlx5_core_dev *dev)
723 {
724 	synchronize_irq(dev->priv.eq_table->cmd_eq.core.irqn);
725 }
726 
727 /* Generic EQ API for mlx5_core consumers
728  * Needed For RDMA ODP EQ for now
729  */
730 struct mlx5_eq *
mlx5_eq_create_generic(struct mlx5_core_dev * dev,struct mlx5_eq_param * param)731 mlx5_eq_create_generic(struct mlx5_core_dev *dev,
732 		       struct mlx5_eq_param *param)
733 {
734 	struct mlx5_eq *eq = kvzalloc_node(sizeof(*eq), GFP_KERNEL,
735 					   dev->priv.numa_node);
736 	int err;
737 
738 	if (!eq)
739 		return ERR_PTR(-ENOMEM);
740 
741 	param->irq = dev->priv.eq_table->ctrl_irq;
742 	err = create_async_eq(dev, eq, param);
743 	if (err) {
744 		kvfree(eq);
745 		eq = ERR_PTR(err);
746 	}
747 
748 	return eq;
749 }
750 EXPORT_SYMBOL(mlx5_eq_create_generic);
751 
mlx5_eq_destroy_generic(struct mlx5_core_dev * dev,struct mlx5_eq * eq)752 int mlx5_eq_destroy_generic(struct mlx5_core_dev *dev, struct mlx5_eq *eq)
753 {
754 	int err;
755 
756 	if (IS_ERR(eq))
757 		return -EINVAL;
758 
759 	err = destroy_async_eq(dev, eq);
760 	if (err)
761 		goto out;
762 
763 	kvfree(eq);
764 out:
765 	return err;
766 }
767 EXPORT_SYMBOL(mlx5_eq_destroy_generic);
768 
mlx5_eq_get_eqe(struct mlx5_eq * eq,u32 cc)769 struct mlx5_eqe *mlx5_eq_get_eqe(struct mlx5_eq *eq, u32 cc)
770 {
771 	u32 ci = eq->cons_index + cc;
772 	u32 nent = eq_get_size(eq);
773 	struct mlx5_eqe *eqe;
774 
775 	eqe = get_eqe(eq, ci & (nent - 1));
776 	eqe = ((eqe->owner & 1) ^ !!(ci & nent)) ? NULL : eqe;
777 	/* Make sure we read EQ entry contents after we've
778 	 * checked the ownership bit.
779 	 */
780 	if (eqe)
781 		dma_rmb();
782 
783 	return eqe;
784 }
785 EXPORT_SYMBOL(mlx5_eq_get_eqe);
786 
mlx5_eq_update_ci(struct mlx5_eq * eq,u32 cc,bool arm)787 void mlx5_eq_update_ci(struct mlx5_eq *eq, u32 cc, bool arm)
788 {
789 	__be32 __iomem *addr = eq->doorbell + (arm ? 0 : 2);
790 	u32 val;
791 
792 	eq->cons_index += cc;
793 	val = (eq->cons_index & 0xffffff) | (eq->eqn << 24);
794 
795 	__raw_writel((__force u32)cpu_to_be32(val), addr);
796 	/* We still want ordering, just not swabbing, so add a barrier */
797 	wmb();
798 }
799 EXPORT_SYMBOL(mlx5_eq_update_ci);
800 
comp_irqs_release(struct mlx5_core_dev * dev)801 static void comp_irqs_release(struct mlx5_core_dev *dev)
802 {
803 	struct mlx5_eq_table *table = dev->priv.eq_table;
804 
805 	if (mlx5_core_is_sf(dev))
806 		mlx5_irq_affinity_irqs_release(dev, table->comp_irqs, table->num_comp_eqs);
807 	else
808 		mlx5_irqs_release_vectors(table->comp_irqs, table->num_comp_eqs);
809 	kfree(table->comp_irqs);
810 }
811 
comp_irqs_request(struct mlx5_core_dev * dev)812 static int comp_irqs_request(struct mlx5_core_dev *dev)
813 {
814 	struct mlx5_eq_table *table = dev->priv.eq_table;
815 	int ncomp_eqs = table->num_comp_eqs;
816 	u16 *cpus;
817 	int ret;
818 	int i;
819 
820 	ncomp_eqs = table->num_comp_eqs;
821 	table->comp_irqs = kcalloc(ncomp_eqs, sizeof(*table->comp_irqs), GFP_KERNEL);
822 	if (!table->comp_irqs)
823 		return -ENOMEM;
824 	if (mlx5_core_is_sf(dev)) {
825 		ret = mlx5_irq_affinity_irqs_request_auto(dev, ncomp_eqs, table->comp_irqs);
826 		if (ret < 0)
827 			goto free_irqs;
828 		return ret;
829 	}
830 
831 	cpus = kcalloc(ncomp_eqs, sizeof(*cpus), GFP_KERNEL);
832 	if (!cpus) {
833 		ret = -ENOMEM;
834 		goto free_irqs;
835 	}
836 	for (i = 0; i < ncomp_eqs; i++)
837 		cpus[i] = cpumask_local_spread(i, dev->priv.numa_node);
838 	ret = mlx5_irqs_request_vectors(dev, cpus, ncomp_eqs, table->comp_irqs);
839 	kfree(cpus);
840 	if (ret < 0)
841 		goto free_irqs;
842 	return ret;
843 
844 free_irqs:
845 	kfree(table->comp_irqs);
846 	return ret;
847 }
848 
destroy_comp_eqs(struct mlx5_core_dev * dev)849 static void destroy_comp_eqs(struct mlx5_core_dev *dev)
850 {
851 	struct mlx5_eq_table *table = dev->priv.eq_table;
852 	struct mlx5_eq_comp *eq, *n;
853 
854 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
855 		list_del(&eq->list);
856 		mlx5_eq_disable(dev, &eq->core, &eq->irq_nb);
857 		if (destroy_unmap_eq(dev, &eq->core))
858 			mlx5_core_warn(dev, "failed to destroy comp EQ 0x%x\n",
859 				       eq->core.eqn);
860 		tasklet_disable(&eq->tasklet_ctx.task);
861 		kfree(eq);
862 	}
863 	comp_irqs_release(dev);
864 }
865 
comp_eq_depth_devlink_param_get(struct mlx5_core_dev * dev)866 static u16 comp_eq_depth_devlink_param_get(struct mlx5_core_dev *dev)
867 {
868 	struct devlink *devlink = priv_to_devlink(dev);
869 	union devlink_param_value val;
870 	int err;
871 
872 	err = devlink_param_driverinit_value_get(devlink,
873 						 DEVLINK_PARAM_GENERIC_ID_IO_EQ_SIZE,
874 						 &val);
875 	if (!err)
876 		return val.vu32;
877 	mlx5_core_dbg(dev, "Failed to get param. using default. err = %d\n", err);
878 	return MLX5_COMP_EQ_SIZE;
879 }
880 
create_comp_eqs(struct mlx5_core_dev * dev)881 static int create_comp_eqs(struct mlx5_core_dev *dev)
882 {
883 	struct mlx5_eq_table *table = dev->priv.eq_table;
884 	struct mlx5_eq_comp *eq;
885 	int ncomp_eqs;
886 	int nent;
887 	int err;
888 	int i;
889 
890 	ncomp_eqs = comp_irqs_request(dev);
891 	if (ncomp_eqs < 0)
892 		return ncomp_eqs;
893 	INIT_LIST_HEAD(&table->comp_eqs_list);
894 	nent = comp_eq_depth_devlink_param_get(dev);
895 
896 	for (i = 0; i < ncomp_eqs; i++) {
897 		struct mlx5_eq_param param = {};
898 
899 		eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, dev->priv.numa_node);
900 		if (!eq) {
901 			err = -ENOMEM;
902 			goto clean;
903 		}
904 
905 		INIT_LIST_HEAD(&eq->tasklet_ctx.list);
906 		INIT_LIST_HEAD(&eq->tasklet_ctx.process_list);
907 		spin_lock_init(&eq->tasklet_ctx.lock);
908 		tasklet_setup(&eq->tasklet_ctx.task, mlx5_cq_tasklet_cb);
909 
910 		eq->irq_nb.notifier_call = mlx5_eq_comp_int;
911 		param = (struct mlx5_eq_param) {
912 			.irq = table->comp_irqs[i],
913 			.nent = nent,
914 		};
915 
916 		err = create_map_eq(dev, &eq->core, &param);
917 		if (err)
918 			goto clean_eq;
919 		err = mlx5_eq_enable(dev, &eq->core, &eq->irq_nb);
920 		if (err) {
921 			destroy_unmap_eq(dev, &eq->core);
922 			goto clean_eq;
923 		}
924 
925 		mlx5_core_dbg(dev, "allocated completion EQN %d\n", eq->core.eqn);
926 		/* add tail, to keep the list ordered, for mlx5_vector2eqn to work */
927 		list_add_tail(&eq->list, &table->comp_eqs_list);
928 	}
929 
930 	table->num_comp_eqs = ncomp_eqs;
931 	return 0;
932 
933 clean_eq:
934 	kfree(eq);
935 clean:
936 	destroy_comp_eqs(dev);
937 	return err;
938 }
939 
vector2eqnirqn(struct mlx5_core_dev * dev,int vector,int * eqn,unsigned int * irqn)940 static int vector2eqnirqn(struct mlx5_core_dev *dev, int vector, int *eqn,
941 			  unsigned int *irqn)
942 {
943 	struct mlx5_eq_table *table = dev->priv.eq_table;
944 	struct mlx5_eq_comp *eq, *n;
945 	int err = -ENOENT;
946 	int i = 0;
947 
948 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
949 		if (i++ == vector) {
950 			if (irqn)
951 				*irqn = eq->core.irqn;
952 			if (eqn)
953 				*eqn = eq->core.eqn;
954 			err = 0;
955 			break;
956 		}
957 	}
958 
959 	return err;
960 }
961 
mlx5_vector2eqn(struct mlx5_core_dev * dev,int vector,int * eqn)962 int mlx5_vector2eqn(struct mlx5_core_dev *dev, int vector, int *eqn)
963 {
964 	return vector2eqnirqn(dev, vector, eqn, NULL);
965 }
966 EXPORT_SYMBOL(mlx5_vector2eqn);
967 
mlx5_vector2irqn(struct mlx5_core_dev * dev,int vector,unsigned int * irqn)968 int mlx5_vector2irqn(struct mlx5_core_dev *dev, int vector, unsigned int *irqn)
969 {
970 	return vector2eqnirqn(dev, vector, NULL, irqn);
971 }
972 
mlx5_comp_vectors_count(struct mlx5_core_dev * dev)973 unsigned int mlx5_comp_vectors_count(struct mlx5_core_dev *dev)
974 {
975 	return dev->priv.eq_table->num_comp_eqs;
976 }
977 EXPORT_SYMBOL(mlx5_comp_vectors_count);
978 
979 struct cpumask *
mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev * dev,int vector)980 mlx5_comp_irq_get_affinity_mask(struct mlx5_core_dev *dev, int vector)
981 {
982 	struct mlx5_eq_table *table = dev->priv.eq_table;
983 	struct mlx5_eq_comp *eq, *n;
984 	int i = 0;
985 
986 	list_for_each_entry_safe(eq, n, &table->comp_eqs_list, list) {
987 		if (i++ == vector)
988 			break;
989 	}
990 
991 	return mlx5_irq_get_affinity_mask(eq->core.irq);
992 }
993 EXPORT_SYMBOL(mlx5_comp_irq_get_affinity_mask);
994 
995 #ifdef CONFIG_RFS_ACCEL
mlx5_eq_table_get_rmap(struct mlx5_core_dev * dev)996 struct cpu_rmap *mlx5_eq_table_get_rmap(struct mlx5_core_dev *dev)
997 {
998 	return dev->priv.eq_table->rmap;
999 }
1000 #endif
1001 
mlx5_eqn2comp_eq(struct mlx5_core_dev * dev,int eqn)1002 struct mlx5_eq_comp *mlx5_eqn2comp_eq(struct mlx5_core_dev *dev, int eqn)
1003 {
1004 	struct mlx5_eq_table *table = dev->priv.eq_table;
1005 	struct mlx5_eq_comp *eq;
1006 
1007 	list_for_each_entry(eq, &table->comp_eqs_list, list) {
1008 		if (eq->core.eqn == eqn)
1009 			return eq;
1010 	}
1011 
1012 	return ERR_PTR(-ENOENT);
1013 }
1014 
clear_rmap(struct mlx5_core_dev * dev)1015 static void clear_rmap(struct mlx5_core_dev *dev)
1016 {
1017 #ifdef CONFIG_RFS_ACCEL
1018 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1019 
1020 	free_irq_cpu_rmap(eq_table->rmap);
1021 #endif
1022 }
1023 
set_rmap(struct mlx5_core_dev * mdev)1024 static int set_rmap(struct mlx5_core_dev *mdev)
1025 {
1026 	int err = 0;
1027 #ifdef CONFIG_RFS_ACCEL
1028 	struct mlx5_eq_table *eq_table = mdev->priv.eq_table;
1029 	int vecidx;
1030 
1031 	eq_table->rmap = alloc_irq_cpu_rmap(eq_table->num_comp_eqs);
1032 	if (!eq_table->rmap) {
1033 		err = -ENOMEM;
1034 		mlx5_core_err(mdev, "Failed to allocate cpu_rmap. err %d", err);
1035 		goto err_out;
1036 	}
1037 
1038 	for (vecidx = 0; vecidx < eq_table->num_comp_eqs; vecidx++) {
1039 		err = irq_cpu_rmap_add(eq_table->rmap,
1040 				       pci_irq_vector(mdev->pdev, vecidx));
1041 		if (err) {
1042 			mlx5_core_err(mdev, "irq_cpu_rmap_add failed. err %d",
1043 				      err);
1044 			goto err_irq_cpu_rmap_add;
1045 		}
1046 	}
1047 	return 0;
1048 
1049 err_irq_cpu_rmap_add:
1050 	clear_rmap(mdev);
1051 err_out:
1052 #endif
1053 	return err;
1054 }
1055 
1056 /* This function should only be called after mlx5_cmd_force_teardown_hca */
mlx5_core_eq_free_irqs(struct mlx5_core_dev * dev)1057 void mlx5_core_eq_free_irqs(struct mlx5_core_dev *dev)
1058 {
1059 	struct mlx5_eq_table *table = dev->priv.eq_table;
1060 
1061 	mutex_lock(&table->lock); /* sync with create/destroy_async_eq */
1062 	if (!mlx5_core_is_sf(dev))
1063 		clear_rmap(dev);
1064 	mlx5_irq_table_destroy(dev);
1065 	mutex_unlock(&table->lock);
1066 }
1067 
1068 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING
1069 #define MLX5_MAX_ASYNC_EQS 4
1070 #else
1071 #define MLX5_MAX_ASYNC_EQS 3
1072 #endif
1073 
mlx5_eq_table_create(struct mlx5_core_dev * dev)1074 int mlx5_eq_table_create(struct mlx5_core_dev *dev)
1075 {
1076 	struct mlx5_eq_table *eq_table = dev->priv.eq_table;
1077 	int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
1078 		      MLX5_CAP_GEN(dev, max_num_eqs) :
1079 		      1 << MLX5_CAP_GEN(dev, log_max_eq);
1080 	int max_eqs_sf;
1081 	int err;
1082 
1083 	eq_table->num_comp_eqs =
1084 		min_t(int,
1085 		      mlx5_irq_table_get_num_comp(eq_table->irq_table),
1086 		      num_eqs - MLX5_MAX_ASYNC_EQS);
1087 	if (mlx5_core_is_sf(dev)) {
1088 		max_eqs_sf = min_t(int, MLX5_COMP_EQS_PER_SF,
1089 				   mlx5_irq_table_get_sfs_vec(eq_table->irq_table));
1090 		eq_table->num_comp_eqs = min_t(int, eq_table->num_comp_eqs,
1091 					       max_eqs_sf);
1092 	}
1093 
1094 	err = create_async_eqs(dev);
1095 	if (err) {
1096 		mlx5_core_err(dev, "Failed to create async EQs\n");
1097 		goto err_async_eqs;
1098 	}
1099 
1100 	if (!mlx5_core_is_sf(dev)) {
1101 		/* rmap is a mapping between irq number and queue number.
1102 		 * each irq can be assign only to a single rmap.
1103 		 * since SFs share IRQs, rmap mapping cannot function correctly
1104 		 * for irqs that are shared for different core/netdev RX rings.
1105 		 * Hence we don't allow netdev rmap for SFs
1106 		 */
1107 		err = set_rmap(dev);
1108 		if (err)
1109 			goto err_rmap;
1110 	}
1111 
1112 	err = create_comp_eqs(dev);
1113 	if (err) {
1114 		mlx5_core_err(dev, "Failed to create completion EQs\n");
1115 		goto err_comp_eqs;
1116 	}
1117 
1118 	return 0;
1119 err_comp_eqs:
1120 	if (!mlx5_core_is_sf(dev))
1121 		clear_rmap(dev);
1122 err_rmap:
1123 	destroy_async_eqs(dev);
1124 err_async_eqs:
1125 	return err;
1126 }
1127 
mlx5_eq_table_destroy(struct mlx5_core_dev * dev)1128 void mlx5_eq_table_destroy(struct mlx5_core_dev *dev)
1129 {
1130 	if (!mlx5_core_is_sf(dev))
1131 		clear_rmap(dev);
1132 	destroy_comp_eqs(dev);
1133 	destroy_async_eqs(dev);
1134 }
1135 
mlx5_eq_notifier_register(struct mlx5_core_dev * dev,struct mlx5_nb * nb)1136 int mlx5_eq_notifier_register(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1137 {
1138 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1139 
1140 	return atomic_notifier_chain_register(&eqt->nh[nb->event_type], &nb->nb);
1141 }
1142 EXPORT_SYMBOL(mlx5_eq_notifier_register);
1143 
mlx5_eq_notifier_unregister(struct mlx5_core_dev * dev,struct mlx5_nb * nb)1144 int mlx5_eq_notifier_unregister(struct mlx5_core_dev *dev, struct mlx5_nb *nb)
1145 {
1146 	struct mlx5_eq_table *eqt = dev->priv.eq_table;
1147 
1148 	return atomic_notifier_chain_unregister(&eqt->nh[nb->event_type], &nb->nb);
1149 }
1150 EXPORT_SYMBOL(mlx5_eq_notifier_unregister);
1151