1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
2 /*
3  * Copyright (c) 2013-2020, Mellanox Technologies inc. All rights reserved.
4  * Copyright (c) 2020, Intel Corporation. All rights reserved.
5  */
6 
7 #include <linux/debugfs.h>
8 #include <linux/highmem.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/errno.h>
12 #include <linux/pci.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/slab.h>
15 #include <linux/bitmap.h>
16 #include <linux/sched.h>
17 #include <linux/sched/mm.h>
18 #include <linux/sched/task.h>
19 #include <linux/delay.h>
20 #include <rdma/ib_user_verbs.h>
21 #include <rdma/ib_addr.h>
22 #include <rdma/ib_cache.h>
23 #include <linux/mlx5/port.h>
24 #include <linux/mlx5/vport.h>
25 #include <linux/mlx5/fs.h>
26 #include <linux/mlx5/eswitch.h>
27 #include <linux/list.h>
28 #include <rdma/ib_smi.h>
29 #include <rdma/ib_umem_odp.h>
30 #include <rdma/lag.h>
31 #include <linux/in.h>
32 #include <linux/etherdevice.h>
33 #include "mlx5_ib.h"
34 #include "ib_rep.h"
35 #include "cmd.h"
36 #include "devx.h"
37 #include "dm.h"
38 #include "fs.h"
39 #include "srq.h"
40 #include "qp.h"
41 #include "wr.h"
42 #include "restrack.h"
43 #include "counters.h"
44 #include "umr.h"
45 #include <rdma/uverbs_std_types.h>
46 #include <rdma/uverbs_ioctl.h>
47 #include <rdma/mlx5_user_ioctl_verbs.h>
48 #include <rdma/mlx5_user_ioctl_cmds.h>
49 
50 #define UVERBS_MODULE_NAME mlx5_ib
51 #include <rdma/uverbs_named_ioctl.h>
52 
53 MODULE_AUTHOR("Eli Cohen <eli@mellanox.com>");
54 MODULE_DESCRIPTION("Mellanox 5th generation network adapters (ConnectX series) IB driver");
55 MODULE_LICENSE("Dual BSD/GPL");
56 
57 struct mlx5_ib_event_work {
58 	struct work_struct	work;
59 	union {
60 		struct mlx5_ib_dev	      *dev;
61 		struct mlx5_ib_multiport_info *mpi;
62 	};
63 	bool			is_slave;
64 	unsigned int		event;
65 	void			*param;
66 };
67 
68 enum {
69 	MLX5_ATOMIC_SIZE_QP_8BYTES = 1 << 3,
70 };
71 
72 static struct workqueue_struct *mlx5_ib_event_wq;
73 static LIST_HEAD(mlx5_ib_unaffiliated_port_list);
74 static LIST_HEAD(mlx5_ib_dev_list);
75 /*
76  * This mutex should be held when accessing either of the above lists
77  */
78 static DEFINE_MUTEX(mlx5_ib_multiport_mutex);
79 
mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info * mpi)80 struct mlx5_ib_dev *mlx5_ib_get_ibdev_from_mpi(struct mlx5_ib_multiport_info *mpi)
81 {
82 	struct mlx5_ib_dev *dev;
83 
84 	mutex_lock(&mlx5_ib_multiport_mutex);
85 	dev = mpi->ibdev;
86 	mutex_unlock(&mlx5_ib_multiport_mutex);
87 	return dev;
88 }
89 
90 static enum rdma_link_layer
mlx5_port_type_cap_to_rdma_ll(int port_type_cap)91 mlx5_port_type_cap_to_rdma_ll(int port_type_cap)
92 {
93 	switch (port_type_cap) {
94 	case MLX5_CAP_PORT_TYPE_IB:
95 		return IB_LINK_LAYER_INFINIBAND;
96 	case MLX5_CAP_PORT_TYPE_ETH:
97 		return IB_LINK_LAYER_ETHERNET;
98 	default:
99 		return IB_LINK_LAYER_UNSPECIFIED;
100 	}
101 }
102 
103 static enum rdma_link_layer
mlx5_ib_port_link_layer(struct ib_device * device,u32 port_num)104 mlx5_ib_port_link_layer(struct ib_device *device, u32 port_num)
105 {
106 	struct mlx5_ib_dev *dev = to_mdev(device);
107 	int port_type_cap = MLX5_CAP_GEN(dev->mdev, port_type);
108 
109 	return mlx5_port_type_cap_to_rdma_ll(port_type_cap);
110 }
111 
get_port_state(struct ib_device * ibdev,u32 port_num,enum ib_port_state * state)112 static int get_port_state(struct ib_device *ibdev,
113 			  u32 port_num,
114 			  enum ib_port_state *state)
115 {
116 	struct ib_port_attr attr;
117 	int ret;
118 
119 	memset(&attr, 0, sizeof(attr));
120 	ret = ibdev->ops.query_port(ibdev, port_num, &attr);
121 	if (!ret)
122 		*state = attr.state;
123 	return ret;
124 }
125 
mlx5_get_rep_roce(struct mlx5_ib_dev * dev,struct net_device * ndev,struct net_device * upper,u32 * port_num)126 static struct mlx5_roce *mlx5_get_rep_roce(struct mlx5_ib_dev *dev,
127 					   struct net_device *ndev,
128 					   struct net_device *upper,
129 					   u32 *port_num)
130 {
131 	struct net_device *rep_ndev;
132 	struct mlx5_ib_port *port;
133 	int i;
134 
135 	for (i = 0; i < dev->num_ports; i++) {
136 		port  = &dev->port[i];
137 		if (!port->rep)
138 			continue;
139 
140 		if (upper == ndev && port->rep->vport == MLX5_VPORT_UPLINK) {
141 			*port_num = i + 1;
142 			return &port->roce;
143 		}
144 
145 		if (upper && port->rep->vport == MLX5_VPORT_UPLINK)
146 			continue;
147 
148 		read_lock(&port->roce.netdev_lock);
149 		rep_ndev = mlx5_ib_get_rep_netdev(port->rep->esw,
150 						  port->rep->vport);
151 		if (rep_ndev == ndev) {
152 			read_unlock(&port->roce.netdev_lock);
153 			*port_num = i + 1;
154 			return &port->roce;
155 		}
156 		read_unlock(&port->roce.netdev_lock);
157 	}
158 
159 	return NULL;
160 }
161 
mlx5_netdev_event(struct notifier_block * this,unsigned long event,void * ptr)162 static int mlx5_netdev_event(struct notifier_block *this,
163 			     unsigned long event, void *ptr)
164 {
165 	struct mlx5_roce *roce = container_of(this, struct mlx5_roce, nb);
166 	struct net_device *ndev = netdev_notifier_info_to_dev(ptr);
167 	u32 port_num = roce->native_port_num;
168 	struct mlx5_core_dev *mdev;
169 	struct mlx5_ib_dev *ibdev;
170 
171 	ibdev = roce->dev;
172 	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
173 	if (!mdev)
174 		return NOTIFY_DONE;
175 
176 	switch (event) {
177 	case NETDEV_REGISTER:
178 		/* Should already be registered during the load */
179 		if (ibdev->is_rep)
180 			break;
181 		write_lock(&roce->netdev_lock);
182 		if (ndev->dev.parent == mdev->device)
183 			roce->netdev = ndev;
184 		write_unlock(&roce->netdev_lock);
185 		break;
186 
187 	case NETDEV_UNREGISTER:
188 		/* In case of reps, ib device goes away before the netdevs */
189 		write_lock(&roce->netdev_lock);
190 		if (roce->netdev == ndev)
191 			roce->netdev = NULL;
192 		write_unlock(&roce->netdev_lock);
193 		break;
194 
195 	case NETDEV_CHANGE:
196 	case NETDEV_UP:
197 	case NETDEV_DOWN: {
198 		struct net_device *lag_ndev = mlx5_lag_get_roce_netdev(mdev);
199 		struct net_device *upper = NULL;
200 
201 		if (lag_ndev) {
202 			upper = netdev_master_upper_dev_get(lag_ndev);
203 			dev_put(lag_ndev);
204 		}
205 
206 		if (ibdev->is_rep)
207 			roce = mlx5_get_rep_roce(ibdev, ndev, upper, &port_num);
208 		if (!roce)
209 			return NOTIFY_DONE;
210 		if ((upper == ndev ||
211 		     ((!upper || ibdev->is_rep) && ndev == roce->netdev)) &&
212 		    ibdev->ib_active) {
213 			struct ib_event ibev = { };
214 			enum ib_port_state port_state;
215 
216 			if (get_port_state(&ibdev->ib_dev, port_num,
217 					   &port_state))
218 				goto done;
219 
220 			if (roce->last_port_state == port_state)
221 				goto done;
222 
223 			roce->last_port_state = port_state;
224 			ibev.device = &ibdev->ib_dev;
225 			if (port_state == IB_PORT_DOWN)
226 				ibev.event = IB_EVENT_PORT_ERR;
227 			else if (port_state == IB_PORT_ACTIVE)
228 				ibev.event = IB_EVENT_PORT_ACTIVE;
229 			else
230 				goto done;
231 
232 			ibev.element.port_num = port_num;
233 			ib_dispatch_event(&ibev);
234 		}
235 		break;
236 	}
237 
238 	default:
239 		break;
240 	}
241 done:
242 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
243 	return NOTIFY_DONE;
244 }
245 
mlx5_ib_get_netdev(struct ib_device * device,u32 port_num)246 static struct net_device *mlx5_ib_get_netdev(struct ib_device *device,
247 					     u32 port_num)
248 {
249 	struct mlx5_ib_dev *ibdev = to_mdev(device);
250 	struct net_device *ndev;
251 	struct mlx5_core_dev *mdev;
252 
253 	mdev = mlx5_ib_get_native_port_mdev(ibdev, port_num, NULL);
254 	if (!mdev)
255 		return NULL;
256 
257 	ndev = mlx5_lag_get_roce_netdev(mdev);
258 	if (ndev)
259 		goto out;
260 
261 	/* Ensure ndev does not disappear before we invoke dev_hold()
262 	 */
263 	read_lock(&ibdev->port[port_num - 1].roce.netdev_lock);
264 	ndev = ibdev->port[port_num - 1].roce.netdev;
265 	if (ndev)
266 		dev_hold(ndev);
267 	read_unlock(&ibdev->port[port_num - 1].roce.netdev_lock);
268 
269 out:
270 	mlx5_ib_put_native_port_mdev(ibdev, port_num);
271 	return ndev;
272 }
273 
mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev * ibdev,u32 ib_port_num,u32 * native_port_num)274 struct mlx5_core_dev *mlx5_ib_get_native_port_mdev(struct mlx5_ib_dev *ibdev,
275 						   u32 ib_port_num,
276 						   u32 *native_port_num)
277 {
278 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
279 							  ib_port_num);
280 	struct mlx5_core_dev *mdev = NULL;
281 	struct mlx5_ib_multiport_info *mpi;
282 	struct mlx5_ib_port *port;
283 
284 	if (!mlx5_core_mp_enabled(ibdev->mdev) ||
285 	    ll != IB_LINK_LAYER_ETHERNET) {
286 		if (native_port_num)
287 			*native_port_num = ib_port_num;
288 		return ibdev->mdev;
289 	}
290 
291 	if (native_port_num)
292 		*native_port_num = 1;
293 
294 	port = &ibdev->port[ib_port_num - 1];
295 	spin_lock(&port->mp.mpi_lock);
296 	mpi = ibdev->port[ib_port_num - 1].mp.mpi;
297 	if (mpi && !mpi->unaffiliate) {
298 		mdev = mpi->mdev;
299 		/* If it's the master no need to refcount, it'll exist
300 		 * as long as the ib_dev exists.
301 		 */
302 		if (!mpi->is_master)
303 			mpi->mdev_refcnt++;
304 	}
305 	spin_unlock(&port->mp.mpi_lock);
306 
307 	return mdev;
308 }
309 
mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev * ibdev,u32 port_num)310 void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *ibdev, u32 port_num)
311 {
312 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&ibdev->ib_dev,
313 							  port_num);
314 	struct mlx5_ib_multiport_info *mpi;
315 	struct mlx5_ib_port *port;
316 
317 	if (!mlx5_core_mp_enabled(ibdev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
318 		return;
319 
320 	port = &ibdev->port[port_num - 1];
321 
322 	spin_lock(&port->mp.mpi_lock);
323 	mpi = ibdev->port[port_num - 1].mp.mpi;
324 	if (mpi->is_master)
325 		goto out;
326 
327 	mpi->mdev_refcnt--;
328 	if (mpi->unaffiliate)
329 		complete(&mpi->unref_comp);
330 out:
331 	spin_unlock(&port->mp.mpi_lock);
332 }
333 
translate_eth_legacy_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width)334 static int translate_eth_legacy_proto_oper(u32 eth_proto_oper,
335 					   u16 *active_speed, u8 *active_width)
336 {
337 	switch (eth_proto_oper) {
338 	case MLX5E_PROT_MASK(MLX5E_1000BASE_CX_SGMII):
339 	case MLX5E_PROT_MASK(MLX5E_1000BASE_KX):
340 	case MLX5E_PROT_MASK(MLX5E_100BASE_TX):
341 	case MLX5E_PROT_MASK(MLX5E_1000BASE_T):
342 		*active_width = IB_WIDTH_1X;
343 		*active_speed = IB_SPEED_SDR;
344 		break;
345 	case MLX5E_PROT_MASK(MLX5E_10GBASE_T):
346 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CX4):
347 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KX4):
348 	case MLX5E_PROT_MASK(MLX5E_10GBASE_KR):
349 	case MLX5E_PROT_MASK(MLX5E_10GBASE_CR):
350 	case MLX5E_PROT_MASK(MLX5E_10GBASE_SR):
351 	case MLX5E_PROT_MASK(MLX5E_10GBASE_ER):
352 		*active_width = IB_WIDTH_1X;
353 		*active_speed = IB_SPEED_QDR;
354 		break;
355 	case MLX5E_PROT_MASK(MLX5E_25GBASE_CR):
356 	case MLX5E_PROT_MASK(MLX5E_25GBASE_KR):
357 	case MLX5E_PROT_MASK(MLX5E_25GBASE_SR):
358 		*active_width = IB_WIDTH_1X;
359 		*active_speed = IB_SPEED_EDR;
360 		break;
361 	case MLX5E_PROT_MASK(MLX5E_40GBASE_CR4):
362 	case MLX5E_PROT_MASK(MLX5E_40GBASE_KR4):
363 	case MLX5E_PROT_MASK(MLX5E_40GBASE_SR4):
364 	case MLX5E_PROT_MASK(MLX5E_40GBASE_LR4):
365 		*active_width = IB_WIDTH_4X;
366 		*active_speed = IB_SPEED_QDR;
367 		break;
368 	case MLX5E_PROT_MASK(MLX5E_50GBASE_CR2):
369 	case MLX5E_PROT_MASK(MLX5E_50GBASE_KR2):
370 	case MLX5E_PROT_MASK(MLX5E_50GBASE_SR2):
371 		*active_width = IB_WIDTH_1X;
372 		*active_speed = IB_SPEED_HDR;
373 		break;
374 	case MLX5E_PROT_MASK(MLX5E_56GBASE_R4):
375 		*active_width = IB_WIDTH_4X;
376 		*active_speed = IB_SPEED_FDR;
377 		break;
378 	case MLX5E_PROT_MASK(MLX5E_100GBASE_CR4):
379 	case MLX5E_PROT_MASK(MLX5E_100GBASE_SR4):
380 	case MLX5E_PROT_MASK(MLX5E_100GBASE_KR4):
381 	case MLX5E_PROT_MASK(MLX5E_100GBASE_LR4):
382 		*active_width = IB_WIDTH_4X;
383 		*active_speed = IB_SPEED_EDR;
384 		break;
385 	default:
386 		return -EINVAL;
387 	}
388 
389 	return 0;
390 }
391 
translate_eth_ext_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width)392 static int translate_eth_ext_proto_oper(u32 eth_proto_oper, u16 *active_speed,
393 					u8 *active_width)
394 {
395 	switch (eth_proto_oper) {
396 	case MLX5E_PROT_MASK(MLX5E_SGMII_100M):
397 	case MLX5E_PROT_MASK(MLX5E_1000BASE_X_SGMII):
398 		*active_width = IB_WIDTH_1X;
399 		*active_speed = IB_SPEED_SDR;
400 		break;
401 	case MLX5E_PROT_MASK(MLX5E_5GBASE_R):
402 		*active_width = IB_WIDTH_1X;
403 		*active_speed = IB_SPEED_DDR;
404 		break;
405 	case MLX5E_PROT_MASK(MLX5E_10GBASE_XFI_XAUI_1):
406 		*active_width = IB_WIDTH_1X;
407 		*active_speed = IB_SPEED_QDR;
408 		break;
409 	case MLX5E_PROT_MASK(MLX5E_40GBASE_XLAUI_4_XLPPI_4):
410 		*active_width = IB_WIDTH_4X;
411 		*active_speed = IB_SPEED_QDR;
412 		break;
413 	case MLX5E_PROT_MASK(MLX5E_25GAUI_1_25GBASE_CR_KR):
414 		*active_width = IB_WIDTH_1X;
415 		*active_speed = IB_SPEED_EDR;
416 		break;
417 	case MLX5E_PROT_MASK(MLX5E_50GAUI_2_LAUI_2_50GBASE_CR2_KR2):
418 		*active_width = IB_WIDTH_2X;
419 		*active_speed = IB_SPEED_EDR;
420 		break;
421 	case MLX5E_PROT_MASK(MLX5E_50GAUI_1_LAUI_1_50GBASE_CR_KR):
422 		*active_width = IB_WIDTH_1X;
423 		*active_speed = IB_SPEED_HDR;
424 		break;
425 	case MLX5E_PROT_MASK(MLX5E_CAUI_4_100GBASE_CR4_KR4):
426 		*active_width = IB_WIDTH_4X;
427 		*active_speed = IB_SPEED_EDR;
428 		break;
429 	case MLX5E_PROT_MASK(MLX5E_100GAUI_2_100GBASE_CR2_KR2):
430 		*active_width = IB_WIDTH_2X;
431 		*active_speed = IB_SPEED_HDR;
432 		break;
433 	case MLX5E_PROT_MASK(MLX5E_100GAUI_1_100GBASE_CR_KR):
434 		*active_width = IB_WIDTH_1X;
435 		*active_speed = IB_SPEED_NDR;
436 		break;
437 	case MLX5E_PROT_MASK(MLX5E_200GAUI_4_200GBASE_CR4_KR4):
438 		*active_width = IB_WIDTH_4X;
439 		*active_speed = IB_SPEED_HDR;
440 		break;
441 	case MLX5E_PROT_MASK(MLX5E_200GAUI_2_200GBASE_CR2_KR2):
442 		*active_width = IB_WIDTH_2X;
443 		*active_speed = IB_SPEED_NDR;
444 		break;
445 	case MLX5E_PROT_MASK(MLX5E_400GAUI_4_400GBASE_CR4_KR4):
446 		*active_width = IB_WIDTH_4X;
447 		*active_speed = IB_SPEED_NDR;
448 		break;
449 	default:
450 		return -EINVAL;
451 	}
452 
453 	return 0;
454 }
455 
translate_eth_proto_oper(u32 eth_proto_oper,u16 * active_speed,u8 * active_width,bool ext)456 static int translate_eth_proto_oper(u32 eth_proto_oper, u16 *active_speed,
457 				    u8 *active_width, bool ext)
458 {
459 	return ext ?
460 		translate_eth_ext_proto_oper(eth_proto_oper, active_speed,
461 					     active_width) :
462 		translate_eth_legacy_proto_oper(eth_proto_oper, active_speed,
463 						active_width);
464 }
465 
mlx5_query_port_roce(struct ib_device * device,u32 port_num,struct ib_port_attr * props)466 static int mlx5_query_port_roce(struct ib_device *device, u32 port_num,
467 				struct ib_port_attr *props)
468 {
469 	struct mlx5_ib_dev *dev = to_mdev(device);
470 	u32 out[MLX5_ST_SZ_DW(ptys_reg)] = {0};
471 	struct mlx5_core_dev *mdev;
472 	struct net_device *ndev, *upper;
473 	enum ib_mtu ndev_ib_mtu;
474 	bool put_mdev = true;
475 	u32 eth_prot_oper;
476 	u32 mdev_port_num;
477 	bool ext;
478 	int err;
479 
480 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
481 	if (!mdev) {
482 		/* This means the port isn't affiliated yet. Get the
483 		 * info for the master port instead.
484 		 */
485 		put_mdev = false;
486 		mdev = dev->mdev;
487 		mdev_port_num = 1;
488 		port_num = 1;
489 	}
490 
491 	/* Possible bad flows are checked before filling out props so in case
492 	 * of an error it will still be zeroed out.
493 	 * Use native port in case of reps
494 	 */
495 	if (dev->is_rep)
496 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
497 					   1);
498 	else
499 		err = mlx5_query_port_ptys(mdev, out, sizeof(out), MLX5_PTYS_EN,
500 					   mdev_port_num);
501 	if (err)
502 		goto out;
503 	ext = !!MLX5_GET_ETH_PROTO(ptys_reg, out, true, eth_proto_capability);
504 	eth_prot_oper = MLX5_GET_ETH_PROTO(ptys_reg, out, ext, eth_proto_oper);
505 
506 	props->active_width     = IB_WIDTH_4X;
507 	props->active_speed     = IB_SPEED_QDR;
508 
509 	translate_eth_proto_oper(eth_prot_oper, &props->active_speed,
510 				 &props->active_width, ext);
511 
512 	if (!dev->is_rep && dev->mdev->roce.roce_en) {
513 		u16 qkey_viol_cntr;
514 
515 		props->port_cap_flags |= IB_PORT_CM_SUP;
516 		props->ip_gids = true;
517 		props->gid_tbl_len = MLX5_CAP_ROCE(dev->mdev,
518 						   roce_address_table_size);
519 		mlx5_query_nic_vport_qkey_viol_cntr(mdev, &qkey_viol_cntr);
520 		props->qkey_viol_cntr = qkey_viol_cntr;
521 	}
522 	props->max_mtu          = IB_MTU_4096;
523 	props->max_msg_sz       = 1 << MLX5_CAP_GEN(dev->mdev, log_max_msg);
524 	props->pkey_tbl_len     = 1;
525 	props->state            = IB_PORT_DOWN;
526 	props->phys_state       = IB_PORT_PHYS_STATE_DISABLED;
527 
528 	/* If this is a stub query for an unaffiliated port stop here */
529 	if (!put_mdev)
530 		goto out;
531 
532 	ndev = mlx5_ib_get_netdev(device, port_num);
533 	if (!ndev)
534 		goto out;
535 
536 	if (dev->lag_active) {
537 		rcu_read_lock();
538 		upper = netdev_master_upper_dev_get_rcu(ndev);
539 		if (upper) {
540 			dev_put(ndev);
541 			ndev = upper;
542 			dev_hold(ndev);
543 		}
544 		rcu_read_unlock();
545 	}
546 
547 	if (netif_running(ndev) && netif_carrier_ok(ndev)) {
548 		props->state      = IB_PORT_ACTIVE;
549 		props->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
550 	}
551 
552 	ndev_ib_mtu = iboe_get_mtu(ndev->mtu);
553 
554 	dev_put(ndev);
555 
556 	props->active_mtu	= min(props->max_mtu, ndev_ib_mtu);
557 out:
558 	if (put_mdev)
559 		mlx5_ib_put_native_port_mdev(dev, port_num);
560 	return err;
561 }
562 
set_roce_addr(struct mlx5_ib_dev * dev,u32 port_num,unsigned int index,const union ib_gid * gid,const struct ib_gid_attr * attr)563 static int set_roce_addr(struct mlx5_ib_dev *dev, u32 port_num,
564 			 unsigned int index, const union ib_gid *gid,
565 			 const struct ib_gid_attr *attr)
566 {
567 	enum ib_gid_type gid_type;
568 	u16 vlan_id = 0xffff;
569 	u8 roce_version = 0;
570 	u8 roce_l3_type = 0;
571 	u8 mac[ETH_ALEN];
572 	int ret;
573 
574 	gid_type = attr->gid_type;
575 	if (gid) {
576 		ret = rdma_read_gid_l2_fields(attr, &vlan_id, &mac[0]);
577 		if (ret)
578 			return ret;
579 	}
580 
581 	switch (gid_type) {
582 	case IB_GID_TYPE_ROCE:
583 		roce_version = MLX5_ROCE_VERSION_1;
584 		break;
585 	case IB_GID_TYPE_ROCE_UDP_ENCAP:
586 		roce_version = MLX5_ROCE_VERSION_2;
587 		if (gid && ipv6_addr_v4mapped((void *)gid))
588 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV4;
589 		else
590 			roce_l3_type = MLX5_ROCE_L3_TYPE_IPV6;
591 		break;
592 
593 	default:
594 		mlx5_ib_warn(dev, "Unexpected GID type %u\n", gid_type);
595 	}
596 
597 	return mlx5_core_roce_gid_set(dev->mdev, index, roce_version,
598 				      roce_l3_type, gid->raw, mac,
599 				      vlan_id < VLAN_CFI_MASK, vlan_id,
600 				      port_num);
601 }
602 
mlx5_ib_add_gid(const struct ib_gid_attr * attr,__always_unused void ** context)603 static int mlx5_ib_add_gid(const struct ib_gid_attr *attr,
604 			   __always_unused void **context)
605 {
606 	return set_roce_addr(to_mdev(attr->device), attr->port_num,
607 			     attr->index, &attr->gid, attr);
608 }
609 
mlx5_ib_del_gid(const struct ib_gid_attr * attr,__always_unused void ** context)610 static int mlx5_ib_del_gid(const struct ib_gid_attr *attr,
611 			   __always_unused void **context)
612 {
613 	return set_roce_addr(to_mdev(attr->device), attr->port_num,
614 			     attr->index, NULL, attr);
615 }
616 
mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev * dev,const struct ib_gid_attr * attr)617 __be16 mlx5_get_roce_udp_sport_min(const struct mlx5_ib_dev *dev,
618 				   const struct ib_gid_attr *attr)
619 {
620 	if (attr->gid_type != IB_GID_TYPE_ROCE_UDP_ENCAP)
621 		return 0;
622 
623 	return cpu_to_be16(MLX5_CAP_ROCE(dev->mdev, r_roce_min_src_udp_port));
624 }
625 
mlx5_use_mad_ifc(struct mlx5_ib_dev * dev)626 static int mlx5_use_mad_ifc(struct mlx5_ib_dev *dev)
627 {
628 	if (MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_IB)
629 		return !MLX5_CAP_GEN(dev->mdev, ib_virt);
630 	return 0;
631 }
632 
633 enum {
634 	MLX5_VPORT_ACCESS_METHOD_MAD,
635 	MLX5_VPORT_ACCESS_METHOD_HCA,
636 	MLX5_VPORT_ACCESS_METHOD_NIC,
637 };
638 
mlx5_get_vport_access_method(struct ib_device * ibdev)639 static int mlx5_get_vport_access_method(struct ib_device *ibdev)
640 {
641 	if (mlx5_use_mad_ifc(to_mdev(ibdev)))
642 		return MLX5_VPORT_ACCESS_METHOD_MAD;
643 
644 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
645 	    IB_LINK_LAYER_ETHERNET)
646 		return MLX5_VPORT_ACCESS_METHOD_NIC;
647 
648 	return MLX5_VPORT_ACCESS_METHOD_HCA;
649 }
650 
get_atomic_caps(struct mlx5_ib_dev * dev,u8 atomic_size_qp,struct ib_device_attr * props)651 static void get_atomic_caps(struct mlx5_ib_dev *dev,
652 			    u8 atomic_size_qp,
653 			    struct ib_device_attr *props)
654 {
655 	u8 tmp;
656 	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
657 	u8 atomic_req_8B_endianness_mode =
658 		MLX5_CAP_ATOMIC(dev->mdev, atomic_req_8B_endianness_mode);
659 
660 	/* Check if HW supports 8 bytes standard atomic operations and capable
661 	 * of host endianness respond
662 	 */
663 	tmp = MLX5_ATOMIC_OPS_CMP_SWAP | MLX5_ATOMIC_OPS_FETCH_ADD;
664 	if (((atomic_operations & tmp) == tmp) &&
665 	    (atomic_size_qp & MLX5_ATOMIC_SIZE_QP_8BYTES) &&
666 	    (atomic_req_8B_endianness_mode)) {
667 		props->atomic_cap = IB_ATOMIC_HCA;
668 	} else {
669 		props->atomic_cap = IB_ATOMIC_NONE;
670 	}
671 }
672 
get_atomic_caps_qp(struct mlx5_ib_dev * dev,struct ib_device_attr * props)673 static void get_atomic_caps_qp(struct mlx5_ib_dev *dev,
674 			       struct ib_device_attr *props)
675 {
676 	u8 atomic_size_qp = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
677 
678 	get_atomic_caps(dev, atomic_size_qp, props);
679 }
680 
mlx5_query_system_image_guid(struct ib_device * ibdev,__be64 * sys_image_guid)681 static int mlx5_query_system_image_guid(struct ib_device *ibdev,
682 					__be64 *sys_image_guid)
683 {
684 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
685 	struct mlx5_core_dev *mdev = dev->mdev;
686 	u64 tmp;
687 	int err;
688 
689 	switch (mlx5_get_vport_access_method(ibdev)) {
690 	case MLX5_VPORT_ACCESS_METHOD_MAD:
691 		return mlx5_query_mad_ifc_system_image_guid(ibdev,
692 							    sys_image_guid);
693 
694 	case MLX5_VPORT_ACCESS_METHOD_HCA:
695 		err = mlx5_query_hca_vport_system_image_guid(mdev, &tmp);
696 		break;
697 
698 	case MLX5_VPORT_ACCESS_METHOD_NIC:
699 		err = mlx5_query_nic_vport_system_image_guid(mdev, &tmp);
700 		break;
701 
702 	default:
703 		return -EINVAL;
704 	}
705 
706 	if (!err)
707 		*sys_image_guid = cpu_to_be64(tmp);
708 
709 	return err;
710 
711 }
712 
mlx5_query_max_pkeys(struct ib_device * ibdev,u16 * max_pkeys)713 static int mlx5_query_max_pkeys(struct ib_device *ibdev,
714 				u16 *max_pkeys)
715 {
716 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
717 	struct mlx5_core_dev *mdev = dev->mdev;
718 
719 	switch (mlx5_get_vport_access_method(ibdev)) {
720 	case MLX5_VPORT_ACCESS_METHOD_MAD:
721 		return mlx5_query_mad_ifc_max_pkeys(ibdev, max_pkeys);
722 
723 	case MLX5_VPORT_ACCESS_METHOD_HCA:
724 	case MLX5_VPORT_ACCESS_METHOD_NIC:
725 		*max_pkeys = mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev,
726 						pkey_table_size));
727 		return 0;
728 
729 	default:
730 		return -EINVAL;
731 	}
732 }
733 
mlx5_query_vendor_id(struct ib_device * ibdev,u32 * vendor_id)734 static int mlx5_query_vendor_id(struct ib_device *ibdev,
735 				u32 *vendor_id)
736 {
737 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
738 
739 	switch (mlx5_get_vport_access_method(ibdev)) {
740 	case MLX5_VPORT_ACCESS_METHOD_MAD:
741 		return mlx5_query_mad_ifc_vendor_id(ibdev, vendor_id);
742 
743 	case MLX5_VPORT_ACCESS_METHOD_HCA:
744 	case MLX5_VPORT_ACCESS_METHOD_NIC:
745 		return mlx5_core_query_vendor_id(dev->mdev, vendor_id);
746 
747 	default:
748 		return -EINVAL;
749 	}
750 }
751 
mlx5_query_node_guid(struct mlx5_ib_dev * dev,__be64 * node_guid)752 static int mlx5_query_node_guid(struct mlx5_ib_dev *dev,
753 				__be64 *node_guid)
754 {
755 	u64 tmp;
756 	int err;
757 
758 	switch (mlx5_get_vport_access_method(&dev->ib_dev)) {
759 	case MLX5_VPORT_ACCESS_METHOD_MAD:
760 		return mlx5_query_mad_ifc_node_guid(dev, node_guid);
761 
762 	case MLX5_VPORT_ACCESS_METHOD_HCA:
763 		err = mlx5_query_hca_vport_node_guid(dev->mdev, &tmp);
764 		break;
765 
766 	case MLX5_VPORT_ACCESS_METHOD_NIC:
767 		err = mlx5_query_nic_vport_node_guid(dev->mdev, &tmp);
768 		break;
769 
770 	default:
771 		return -EINVAL;
772 	}
773 
774 	if (!err)
775 		*node_guid = cpu_to_be64(tmp);
776 
777 	return err;
778 }
779 
780 struct mlx5_reg_node_desc {
781 	u8	desc[IB_DEVICE_NODE_DESC_MAX];
782 };
783 
mlx5_query_node_desc(struct mlx5_ib_dev * dev,char * node_desc)784 static int mlx5_query_node_desc(struct mlx5_ib_dev *dev, char *node_desc)
785 {
786 	struct mlx5_reg_node_desc in;
787 
788 	if (mlx5_use_mad_ifc(dev))
789 		return mlx5_query_mad_ifc_node_desc(dev, node_desc);
790 
791 	memset(&in, 0, sizeof(in));
792 
793 	return mlx5_core_access_reg(dev->mdev, &in, sizeof(in), node_desc,
794 				    sizeof(struct mlx5_reg_node_desc),
795 				    MLX5_REG_NODE_DESC, 0, 0);
796 }
797 
mlx5_ib_query_device(struct ib_device * ibdev,struct ib_device_attr * props,struct ib_udata * uhw)798 static int mlx5_ib_query_device(struct ib_device *ibdev,
799 				struct ib_device_attr *props,
800 				struct ib_udata *uhw)
801 {
802 	size_t uhw_outlen = (uhw) ? uhw->outlen : 0;
803 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
804 	struct mlx5_core_dev *mdev = dev->mdev;
805 	int err = -ENOMEM;
806 	int max_sq_desc;
807 	int max_rq_sg;
808 	int max_sq_sg;
809 	u64 min_page_size = 1ull << MLX5_CAP_GEN(mdev, log_pg_sz);
810 	bool raw_support = !mlx5_core_mp_enabled(mdev);
811 	struct mlx5_ib_query_device_resp resp = {};
812 	size_t resp_len;
813 	u64 max_tso;
814 
815 	resp_len = sizeof(resp.comp_mask) + sizeof(resp.response_length);
816 	if (uhw_outlen && uhw_outlen < resp_len)
817 		return -EINVAL;
818 
819 	resp.response_length = resp_len;
820 
821 	if (uhw && uhw->inlen && !ib_is_udata_cleared(uhw, 0, uhw->inlen))
822 		return -EINVAL;
823 
824 	memset(props, 0, sizeof(*props));
825 	err = mlx5_query_system_image_guid(ibdev,
826 					   &props->sys_image_guid);
827 	if (err)
828 		return err;
829 
830 	props->max_pkeys = dev->pkey_table_len;
831 
832 	err = mlx5_query_vendor_id(ibdev, &props->vendor_id);
833 	if (err)
834 		return err;
835 
836 	props->fw_ver = ((u64)fw_rev_maj(dev->mdev) << 32) |
837 		(fw_rev_min(dev->mdev) << 16) |
838 		fw_rev_sub(dev->mdev);
839 	props->device_cap_flags    = IB_DEVICE_CHANGE_PHY_PORT |
840 		IB_DEVICE_PORT_ACTIVE_EVENT		|
841 		IB_DEVICE_SYS_IMAGE_GUID		|
842 		IB_DEVICE_RC_RNR_NAK_GEN;
843 
844 	if (MLX5_CAP_GEN(mdev, pkv))
845 		props->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
846 	if (MLX5_CAP_GEN(mdev, qkv))
847 		props->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
848 	if (MLX5_CAP_GEN(mdev, apm))
849 		props->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
850 	if (MLX5_CAP_GEN(mdev, xrc))
851 		props->device_cap_flags |= IB_DEVICE_XRC;
852 	if (MLX5_CAP_GEN(mdev, imaicl)) {
853 		props->device_cap_flags |= IB_DEVICE_MEM_WINDOW |
854 					   IB_DEVICE_MEM_WINDOW_TYPE_2B;
855 		props->max_mw = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
856 		/* We support 'Gappy' memory registration too */
857 		props->kernel_cap_flags |= IBK_SG_GAPS_REG;
858 	}
859 	/* IB_WR_REG_MR always requires changing the entity size with UMR */
860 	if (!MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled))
861 		props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS;
862 	if (MLX5_CAP_GEN(mdev, sho)) {
863 		props->kernel_cap_flags |= IBK_INTEGRITY_HANDOVER;
864 		/* At this stage no support for signature handover */
865 		props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 |
866 				      IB_PROT_T10DIF_TYPE_2 |
867 				      IB_PROT_T10DIF_TYPE_3;
868 		props->sig_guard_cap = IB_GUARD_T10DIF_CRC |
869 				       IB_GUARD_T10DIF_CSUM;
870 	}
871 	if (MLX5_CAP_GEN(mdev, block_lb_mc))
872 		props->kernel_cap_flags |= IBK_BLOCK_MULTICAST_LOOPBACK;
873 
874 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) && raw_support) {
875 		if (MLX5_CAP_ETH(mdev, csum_cap)) {
876 			/* Legacy bit to support old userspace libraries */
877 			props->device_cap_flags |= IB_DEVICE_RAW_IP_CSUM;
878 			props->raw_packet_caps |= IB_RAW_PACKET_CAP_IP_CSUM;
879 		}
880 
881 		if (MLX5_CAP_ETH(dev->mdev, vlan_cap))
882 			props->raw_packet_caps |=
883 				IB_RAW_PACKET_CAP_CVLAN_STRIPPING;
884 
885 		if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen) {
886 			max_tso = MLX5_CAP_ETH(mdev, max_lso_cap);
887 			if (max_tso) {
888 				resp.tso_caps.max_tso = 1 << max_tso;
889 				resp.tso_caps.supported_qpts |=
890 					1 << IB_QPT_RAW_PACKET;
891 				resp.response_length += sizeof(resp.tso_caps);
892 			}
893 		}
894 
895 		if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen) {
896 			resp.rss_caps.rx_hash_function =
897 						MLX5_RX_HASH_FUNC_TOEPLITZ;
898 			resp.rss_caps.rx_hash_fields_mask =
899 						MLX5_RX_HASH_SRC_IPV4 |
900 						MLX5_RX_HASH_DST_IPV4 |
901 						MLX5_RX_HASH_SRC_IPV6 |
902 						MLX5_RX_HASH_DST_IPV6 |
903 						MLX5_RX_HASH_SRC_PORT_TCP |
904 						MLX5_RX_HASH_DST_PORT_TCP |
905 						MLX5_RX_HASH_SRC_PORT_UDP |
906 						MLX5_RX_HASH_DST_PORT_UDP |
907 						MLX5_RX_HASH_INNER;
908 			resp.response_length += sizeof(resp.rss_caps);
909 		}
910 	} else {
911 		if (offsetofend(typeof(resp), tso_caps) <= uhw_outlen)
912 			resp.response_length += sizeof(resp.tso_caps);
913 		if (offsetofend(typeof(resp), rss_caps) <= uhw_outlen)
914 			resp.response_length += sizeof(resp.rss_caps);
915 	}
916 
917 	if (MLX5_CAP_GEN(mdev, ipoib_basic_offloads)) {
918 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
919 		props->kernel_cap_flags |= IBK_UD_TSO;
920 	}
921 
922 	if (MLX5_CAP_GEN(dev->mdev, rq_delay_drop) &&
923 	    MLX5_CAP_GEN(dev->mdev, general_notification_event) &&
924 	    raw_support)
925 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_DELAY_DROP;
926 
927 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
928 	    MLX5_CAP_IPOIB_ENHANCED(mdev, csum_cap))
929 		props->device_cap_flags |= IB_DEVICE_UD_IP_CSUM;
930 
931 	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
932 	    MLX5_CAP_ETH(dev->mdev, scatter_fcs) &&
933 	    raw_support) {
934 		/* Legacy bit to support old userspace libraries */
935 		props->device_cap_flags |= IB_DEVICE_RAW_SCATTER_FCS;
936 		props->raw_packet_caps |= IB_RAW_PACKET_CAP_SCATTER_FCS;
937 	}
938 
939 	if (MLX5_CAP_DEV_MEM(mdev, memic)) {
940 		props->max_dm_size =
941 			MLX5_CAP_DEV_MEM(mdev, max_memic_size);
942 	}
943 
944 	if (mlx5_get_flow_namespace(dev->mdev, MLX5_FLOW_NAMESPACE_BYPASS))
945 		props->device_cap_flags |= IB_DEVICE_MANAGED_FLOW_STEERING;
946 
947 	if (MLX5_CAP_GEN(mdev, end_pad))
948 		props->device_cap_flags |= IB_DEVICE_PCI_WRITE_END_PADDING;
949 
950 	props->vendor_part_id	   = mdev->pdev->device;
951 	props->hw_ver		   = mdev->pdev->revision;
952 
953 	props->max_mr_size	   = ~0ull;
954 	props->page_size_cap	   = ~(min_page_size - 1);
955 	props->max_qp		   = 1 << MLX5_CAP_GEN(mdev, log_max_qp);
956 	props->max_qp_wr	   = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
957 	max_rq_sg =  MLX5_CAP_GEN(mdev, max_wqe_sz_rq) /
958 		     sizeof(struct mlx5_wqe_data_seg);
959 	max_sq_desc = min_t(int, MLX5_CAP_GEN(mdev, max_wqe_sz_sq), 512);
960 	max_sq_sg = (max_sq_desc - sizeof(struct mlx5_wqe_ctrl_seg) -
961 		     sizeof(struct mlx5_wqe_raddr_seg)) /
962 		sizeof(struct mlx5_wqe_data_seg);
963 	props->max_send_sge = max_sq_sg;
964 	props->max_recv_sge = max_rq_sg;
965 	props->max_sge_rd	   = MLX5_MAX_SGE_RD;
966 	props->max_cq		   = 1 << MLX5_CAP_GEN(mdev, log_max_cq);
967 	props->max_cqe = (1 << MLX5_CAP_GEN(mdev, log_max_cq_sz)) - 1;
968 	props->max_mr		   = 1 << MLX5_CAP_GEN(mdev, log_max_mkey);
969 	props->max_pd		   = 1 << MLX5_CAP_GEN(mdev, log_max_pd);
970 	props->max_qp_rd_atom	   = 1 << MLX5_CAP_GEN(mdev, log_max_ra_req_qp);
971 	props->max_qp_init_rd_atom = 1 << MLX5_CAP_GEN(mdev, log_max_ra_res_qp);
972 	props->max_srq		   = 1 << MLX5_CAP_GEN(mdev, log_max_srq);
973 	props->max_srq_wr = (1 << MLX5_CAP_GEN(mdev, log_max_srq_sz)) - 1;
974 	props->local_ca_ack_delay  = MLX5_CAP_GEN(mdev, local_ca_ack_delay);
975 	props->max_res_rd_atom	   = props->max_qp_rd_atom * props->max_qp;
976 	props->max_srq_sge	   = max_rq_sg - 1;
977 	props->max_fast_reg_page_list_len =
978 		1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size);
979 	props->max_pi_fast_reg_page_list_len =
980 		props->max_fast_reg_page_list_len / 2;
981 	props->max_sgl_rd =
982 		MLX5_CAP_GEN(mdev, max_sgl_for_optimized_performance);
983 	get_atomic_caps_qp(dev, props);
984 	props->masked_atomic_cap   = IB_ATOMIC_NONE;
985 	props->max_mcast_grp	   = 1 << MLX5_CAP_GEN(mdev, log_max_mcg);
986 	props->max_mcast_qp_attach = MLX5_CAP_GEN(mdev, max_qp_mcg);
987 	props->max_total_mcast_qp_attach = props->max_mcast_qp_attach *
988 					   props->max_mcast_grp;
989 	props->max_ah = INT_MAX;
990 	props->hca_core_clock = MLX5_CAP_GEN(mdev, device_frequency_khz);
991 	props->timestamp_mask = 0x7FFFFFFFFFFFFFFFULL;
992 
993 	if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) {
994 		if (dev->odp_caps.general_caps & IB_ODP_SUPPORT)
995 			props->kernel_cap_flags |= IBK_ON_DEMAND_PAGING;
996 		props->odp_caps = dev->odp_caps;
997 		if (!uhw) {
998 			/* ODP for kernel QPs is not implemented for receive
999 			 * WQEs and SRQ WQEs
1000 			 */
1001 			props->odp_caps.per_transport_caps.rc_odp_caps &=
1002 				~(IB_ODP_SUPPORT_READ |
1003 				  IB_ODP_SUPPORT_SRQ_RECV);
1004 			props->odp_caps.per_transport_caps.uc_odp_caps &=
1005 				~(IB_ODP_SUPPORT_READ |
1006 				  IB_ODP_SUPPORT_SRQ_RECV);
1007 			props->odp_caps.per_transport_caps.ud_odp_caps &=
1008 				~(IB_ODP_SUPPORT_READ |
1009 				  IB_ODP_SUPPORT_SRQ_RECV);
1010 			props->odp_caps.per_transport_caps.xrc_odp_caps &=
1011 				~(IB_ODP_SUPPORT_READ |
1012 				  IB_ODP_SUPPORT_SRQ_RECV);
1013 		}
1014 	}
1015 
1016 	if (mlx5_core_is_vf(mdev))
1017 		props->kernel_cap_flags |= IBK_VIRTUAL_FUNCTION;
1018 
1019 	if (mlx5_ib_port_link_layer(ibdev, 1) ==
1020 	    IB_LINK_LAYER_ETHERNET && raw_support) {
1021 		props->rss_caps.max_rwq_indirection_tables =
1022 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt);
1023 		props->rss_caps.max_rwq_indirection_table_size =
1024 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rqt_size);
1025 		props->rss_caps.supported_qpts = 1 << IB_QPT_RAW_PACKET;
1026 		props->max_wq_type_rq =
1027 			1 << MLX5_CAP_GEN(dev->mdev, log_max_rq);
1028 	}
1029 
1030 	if (MLX5_CAP_GEN(mdev, tag_matching)) {
1031 		props->tm_caps.max_num_tags =
1032 			(1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1;
1033 		props->tm_caps.max_ops =
1034 			1 << MLX5_CAP_GEN(mdev, log_max_qp_sz);
1035 		props->tm_caps.max_sge = MLX5_TM_MAX_SGE;
1036 	}
1037 
1038 	if (MLX5_CAP_GEN(mdev, tag_matching) &&
1039 	    MLX5_CAP_GEN(mdev, rndv_offload_rc)) {
1040 		props->tm_caps.flags = IB_TM_CAP_RNDV_RC;
1041 		props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE;
1042 	}
1043 
1044 	if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) {
1045 		props->cq_caps.max_cq_moderation_count =
1046 						MLX5_MAX_CQ_COUNT;
1047 		props->cq_caps.max_cq_moderation_period =
1048 						MLX5_MAX_CQ_PERIOD;
1049 	}
1050 
1051 	if (offsetofend(typeof(resp), cqe_comp_caps) <= uhw_outlen) {
1052 		resp.response_length += sizeof(resp.cqe_comp_caps);
1053 
1054 		if (MLX5_CAP_GEN(dev->mdev, cqe_compression)) {
1055 			resp.cqe_comp_caps.max_num =
1056 				MLX5_CAP_GEN(dev->mdev,
1057 					     cqe_compression_max_num);
1058 
1059 			resp.cqe_comp_caps.supported_format =
1060 				MLX5_IB_CQE_RES_FORMAT_HASH |
1061 				MLX5_IB_CQE_RES_FORMAT_CSUM;
1062 
1063 			if (MLX5_CAP_GEN(dev->mdev, mini_cqe_resp_stride_index))
1064 				resp.cqe_comp_caps.supported_format |=
1065 					MLX5_IB_CQE_RES_FORMAT_CSUM_STRIDX;
1066 		}
1067 	}
1068 
1069 	if (offsetofend(typeof(resp), packet_pacing_caps) <= uhw_outlen &&
1070 	    raw_support) {
1071 		if (MLX5_CAP_QOS(mdev, packet_pacing) &&
1072 		    MLX5_CAP_GEN(mdev, qos)) {
1073 			resp.packet_pacing_caps.qp_rate_limit_max =
1074 				MLX5_CAP_QOS(mdev, packet_pacing_max_rate);
1075 			resp.packet_pacing_caps.qp_rate_limit_min =
1076 				MLX5_CAP_QOS(mdev, packet_pacing_min_rate);
1077 			resp.packet_pacing_caps.supported_qpts |=
1078 				1 << IB_QPT_RAW_PACKET;
1079 			if (MLX5_CAP_QOS(mdev, packet_pacing_burst_bound) &&
1080 			    MLX5_CAP_QOS(mdev, packet_pacing_typical_size))
1081 				resp.packet_pacing_caps.cap_flags |=
1082 					MLX5_IB_PP_SUPPORT_BURST;
1083 		}
1084 		resp.response_length += sizeof(resp.packet_pacing_caps);
1085 	}
1086 
1087 	if (offsetofend(typeof(resp), mlx5_ib_support_multi_pkt_send_wqes) <=
1088 	    uhw_outlen) {
1089 		if (MLX5_CAP_ETH(mdev, multi_pkt_send_wqe))
1090 			resp.mlx5_ib_support_multi_pkt_send_wqes =
1091 				MLX5_IB_ALLOW_MPW;
1092 
1093 		if (MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe))
1094 			resp.mlx5_ib_support_multi_pkt_send_wqes |=
1095 				MLX5_IB_SUPPORT_EMPW;
1096 
1097 		resp.response_length +=
1098 			sizeof(resp.mlx5_ib_support_multi_pkt_send_wqes);
1099 	}
1100 
1101 	if (offsetofend(typeof(resp), flags) <= uhw_outlen) {
1102 		resp.response_length += sizeof(resp.flags);
1103 
1104 		if (MLX5_CAP_GEN(mdev, cqe_compression_128))
1105 			resp.flags |=
1106 				MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_COMP;
1107 
1108 		if (MLX5_CAP_GEN(mdev, cqe_128_always))
1109 			resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_CQE_128B_PAD;
1110 		if (MLX5_CAP_GEN(mdev, qp_packet_based))
1111 			resp.flags |=
1112 				MLX5_IB_QUERY_DEV_RESP_PACKET_BASED_CREDIT_MODE;
1113 
1114 		resp.flags |= MLX5_IB_QUERY_DEV_RESP_FLAGS_SCAT2CQE_DCT;
1115 	}
1116 
1117 	if (offsetofend(typeof(resp), sw_parsing_caps) <= uhw_outlen) {
1118 		resp.response_length += sizeof(resp.sw_parsing_caps);
1119 		if (MLX5_CAP_ETH(mdev, swp)) {
1120 			resp.sw_parsing_caps.sw_parsing_offloads |=
1121 				MLX5_IB_SW_PARSING;
1122 
1123 			if (MLX5_CAP_ETH(mdev, swp_csum))
1124 				resp.sw_parsing_caps.sw_parsing_offloads |=
1125 					MLX5_IB_SW_PARSING_CSUM;
1126 
1127 			if (MLX5_CAP_ETH(mdev, swp_lso))
1128 				resp.sw_parsing_caps.sw_parsing_offloads |=
1129 					MLX5_IB_SW_PARSING_LSO;
1130 
1131 			if (resp.sw_parsing_caps.sw_parsing_offloads)
1132 				resp.sw_parsing_caps.supported_qpts =
1133 					BIT(IB_QPT_RAW_PACKET);
1134 		}
1135 	}
1136 
1137 	if (offsetofend(typeof(resp), striding_rq_caps) <= uhw_outlen &&
1138 	    raw_support) {
1139 		resp.response_length += sizeof(resp.striding_rq_caps);
1140 		if (MLX5_CAP_GEN(mdev, striding_rq)) {
1141 			resp.striding_rq_caps.min_single_stride_log_num_of_bytes =
1142 				MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES;
1143 			resp.striding_rq_caps.max_single_stride_log_num_of_bytes =
1144 				MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES;
1145 			if (MLX5_CAP_GEN(dev->mdev, ext_stride_num_range))
1146 				resp.striding_rq_caps
1147 					.min_single_wqe_log_num_of_strides =
1148 					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1149 			else
1150 				resp.striding_rq_caps
1151 					.min_single_wqe_log_num_of_strides =
1152 					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES;
1153 			resp.striding_rq_caps.max_single_wqe_log_num_of_strides =
1154 				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES;
1155 			resp.striding_rq_caps.supported_qpts =
1156 				BIT(IB_QPT_RAW_PACKET);
1157 		}
1158 	}
1159 
1160 	if (offsetofend(typeof(resp), tunnel_offloads_caps) <= uhw_outlen) {
1161 		resp.response_length += sizeof(resp.tunnel_offloads_caps);
1162 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan))
1163 			resp.tunnel_offloads_caps |=
1164 				MLX5_IB_TUNNELED_OFFLOADS_VXLAN;
1165 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx))
1166 			resp.tunnel_offloads_caps |=
1167 				MLX5_IB_TUNNELED_OFFLOADS_GENEVE;
1168 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre))
1169 			resp.tunnel_offloads_caps |=
1170 				MLX5_IB_TUNNELED_OFFLOADS_GRE;
1171 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_gre))
1172 			resp.tunnel_offloads_caps |=
1173 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_GRE;
1174 		if (MLX5_CAP_ETH(mdev, tunnel_stateless_mpls_over_udp))
1175 			resp.tunnel_offloads_caps |=
1176 				MLX5_IB_TUNNELED_OFFLOADS_MPLS_UDP;
1177 	}
1178 
1179 	if (offsetofend(typeof(resp), dci_streams_caps) <= uhw_outlen) {
1180 		resp.response_length += sizeof(resp.dci_streams_caps);
1181 
1182 		resp.dci_streams_caps.max_log_num_concurent =
1183 			MLX5_CAP_GEN(mdev, log_max_dci_stream_channels);
1184 
1185 		resp.dci_streams_caps.max_log_num_errored =
1186 			MLX5_CAP_GEN(mdev, log_max_dci_errored_streams);
1187 	}
1188 
1189 	if (uhw_outlen) {
1190 		err = ib_copy_to_udata(uhw, &resp, resp.response_length);
1191 
1192 		if (err)
1193 			return err;
1194 	}
1195 
1196 	return 0;
1197 }
1198 
translate_active_width(struct ib_device * ibdev,u16 active_width,u8 * ib_width)1199 static void translate_active_width(struct ib_device *ibdev, u16 active_width,
1200 				   u8 *ib_width)
1201 {
1202 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1203 
1204 	if (active_width & MLX5_PTYS_WIDTH_1X)
1205 		*ib_width = IB_WIDTH_1X;
1206 	else if (active_width & MLX5_PTYS_WIDTH_2X)
1207 		*ib_width = IB_WIDTH_2X;
1208 	else if (active_width & MLX5_PTYS_WIDTH_4X)
1209 		*ib_width = IB_WIDTH_4X;
1210 	else if (active_width & MLX5_PTYS_WIDTH_8X)
1211 		*ib_width = IB_WIDTH_8X;
1212 	else if (active_width & MLX5_PTYS_WIDTH_12X)
1213 		*ib_width = IB_WIDTH_12X;
1214 	else {
1215 		mlx5_ib_dbg(dev, "Invalid active_width %d, setting width to default value: 4x\n",
1216 			    active_width);
1217 		*ib_width = IB_WIDTH_4X;
1218 	}
1219 
1220 	return;
1221 }
1222 
mlx5_mtu_to_ib_mtu(int mtu)1223 static int mlx5_mtu_to_ib_mtu(int mtu)
1224 {
1225 	switch (mtu) {
1226 	case 256: return 1;
1227 	case 512: return 2;
1228 	case 1024: return 3;
1229 	case 2048: return 4;
1230 	case 4096: return 5;
1231 	default:
1232 		pr_warn("invalid mtu\n");
1233 		return -1;
1234 	}
1235 }
1236 
1237 enum ib_max_vl_num {
1238 	__IB_MAX_VL_0		= 1,
1239 	__IB_MAX_VL_0_1		= 2,
1240 	__IB_MAX_VL_0_3		= 3,
1241 	__IB_MAX_VL_0_7		= 4,
1242 	__IB_MAX_VL_0_14	= 5,
1243 };
1244 
1245 enum mlx5_vl_hw_cap {
1246 	MLX5_VL_HW_0	= 1,
1247 	MLX5_VL_HW_0_1	= 2,
1248 	MLX5_VL_HW_0_2	= 3,
1249 	MLX5_VL_HW_0_3	= 4,
1250 	MLX5_VL_HW_0_4	= 5,
1251 	MLX5_VL_HW_0_5	= 6,
1252 	MLX5_VL_HW_0_6	= 7,
1253 	MLX5_VL_HW_0_7	= 8,
1254 	MLX5_VL_HW_0_14	= 15
1255 };
1256 
translate_max_vl_num(struct ib_device * ibdev,u8 vl_hw_cap,u8 * max_vl_num)1257 static int translate_max_vl_num(struct ib_device *ibdev, u8 vl_hw_cap,
1258 				u8 *max_vl_num)
1259 {
1260 	switch (vl_hw_cap) {
1261 	case MLX5_VL_HW_0:
1262 		*max_vl_num = __IB_MAX_VL_0;
1263 		break;
1264 	case MLX5_VL_HW_0_1:
1265 		*max_vl_num = __IB_MAX_VL_0_1;
1266 		break;
1267 	case MLX5_VL_HW_0_3:
1268 		*max_vl_num = __IB_MAX_VL_0_3;
1269 		break;
1270 	case MLX5_VL_HW_0_7:
1271 		*max_vl_num = __IB_MAX_VL_0_7;
1272 		break;
1273 	case MLX5_VL_HW_0_14:
1274 		*max_vl_num = __IB_MAX_VL_0_14;
1275 		break;
1276 
1277 	default:
1278 		return -EINVAL;
1279 	}
1280 
1281 	return 0;
1282 }
1283 
mlx5_query_hca_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1284 static int mlx5_query_hca_port(struct ib_device *ibdev, u32 port,
1285 			       struct ib_port_attr *props)
1286 {
1287 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1288 	struct mlx5_core_dev *mdev = dev->mdev;
1289 	struct mlx5_hca_vport_context *rep;
1290 	u16 max_mtu;
1291 	u16 oper_mtu;
1292 	int err;
1293 	u16 ib_link_width_oper;
1294 	u8 vl_hw_cap;
1295 
1296 	rep = kzalloc(sizeof(*rep), GFP_KERNEL);
1297 	if (!rep) {
1298 		err = -ENOMEM;
1299 		goto out;
1300 	}
1301 
1302 	/* props being zeroed by the caller, avoid zeroing it here */
1303 
1304 	err = mlx5_query_hca_vport_context(mdev, 0, port, 0, rep);
1305 	if (err)
1306 		goto out;
1307 
1308 	props->lid		= rep->lid;
1309 	props->lmc		= rep->lmc;
1310 	props->sm_lid		= rep->sm_lid;
1311 	props->sm_sl		= rep->sm_sl;
1312 	props->state		= rep->vport_state;
1313 	props->phys_state	= rep->port_physical_state;
1314 	props->port_cap_flags	= rep->cap_mask1;
1315 	props->gid_tbl_len	= mlx5_get_gid_table_len(MLX5_CAP_GEN(mdev, gid_table_size));
1316 	props->max_msg_sz	= 1 << MLX5_CAP_GEN(mdev, log_max_msg);
1317 	props->pkey_tbl_len	= mlx5_to_sw_pkey_sz(MLX5_CAP_GEN(mdev, pkey_table_size));
1318 	props->bad_pkey_cntr	= rep->pkey_violation_counter;
1319 	props->qkey_viol_cntr	= rep->qkey_violation_counter;
1320 	props->subnet_timeout	= rep->subnet_timeout;
1321 	props->init_type_reply	= rep->init_type_reply;
1322 
1323 	if (props->port_cap_flags & IB_PORT_CAP_MASK2_SUP)
1324 		props->port_cap_flags2 = rep->cap_mask2;
1325 
1326 	err = mlx5_query_ib_port_oper(mdev, &ib_link_width_oper,
1327 				      &props->active_speed, port);
1328 	if (err)
1329 		goto out;
1330 
1331 	translate_active_width(ibdev, ib_link_width_oper, &props->active_width);
1332 
1333 	mlx5_query_port_max_mtu(mdev, &max_mtu, port);
1334 
1335 	props->max_mtu = mlx5_mtu_to_ib_mtu(max_mtu);
1336 
1337 	mlx5_query_port_oper_mtu(mdev, &oper_mtu, port);
1338 
1339 	props->active_mtu = mlx5_mtu_to_ib_mtu(oper_mtu);
1340 
1341 	err = mlx5_query_port_vl_hw_cap(mdev, &vl_hw_cap, port);
1342 	if (err)
1343 		goto out;
1344 
1345 	err = translate_max_vl_num(ibdev, vl_hw_cap,
1346 				   &props->max_vl_num);
1347 out:
1348 	kfree(rep);
1349 	return err;
1350 }
1351 
mlx5_ib_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1352 int mlx5_ib_query_port(struct ib_device *ibdev, u32 port,
1353 		       struct ib_port_attr *props)
1354 {
1355 	unsigned int count;
1356 	int ret;
1357 
1358 	switch (mlx5_get_vport_access_method(ibdev)) {
1359 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1360 		ret = mlx5_query_mad_ifc_port(ibdev, port, props);
1361 		break;
1362 
1363 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1364 		ret = mlx5_query_hca_port(ibdev, port, props);
1365 		break;
1366 
1367 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1368 		ret = mlx5_query_port_roce(ibdev, port, props);
1369 		break;
1370 
1371 	default:
1372 		ret = -EINVAL;
1373 	}
1374 
1375 	if (!ret && props) {
1376 		struct mlx5_ib_dev *dev = to_mdev(ibdev);
1377 		struct mlx5_core_dev *mdev;
1378 		bool put_mdev = true;
1379 
1380 		mdev = mlx5_ib_get_native_port_mdev(dev, port, NULL);
1381 		if (!mdev) {
1382 			/* If the port isn't affiliated yet query the master.
1383 			 * The master and slave will have the same values.
1384 			 */
1385 			mdev = dev->mdev;
1386 			port = 1;
1387 			put_mdev = false;
1388 		}
1389 		count = mlx5_core_reserved_gids_count(mdev);
1390 		if (put_mdev)
1391 			mlx5_ib_put_native_port_mdev(dev, port);
1392 		props->gid_tbl_len -= count;
1393 	}
1394 	return ret;
1395 }
1396 
mlx5_ib_rep_query_port(struct ib_device * ibdev,u32 port,struct ib_port_attr * props)1397 static int mlx5_ib_rep_query_port(struct ib_device *ibdev, u32 port,
1398 				  struct ib_port_attr *props)
1399 {
1400 	return mlx5_query_port_roce(ibdev, port, props);
1401 }
1402 
mlx5_ib_rep_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1403 static int mlx5_ib_rep_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1404 				  u16 *pkey)
1405 {
1406 	/* Default special Pkey for representor device port as per the
1407 	 * IB specification 1.3 section 10.9.1.2.
1408 	 */
1409 	*pkey = 0xffff;
1410 	return 0;
1411 }
1412 
mlx5_ib_query_gid(struct ib_device * ibdev,u32 port,int index,union ib_gid * gid)1413 static int mlx5_ib_query_gid(struct ib_device *ibdev, u32 port, int index,
1414 			     union ib_gid *gid)
1415 {
1416 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1417 	struct mlx5_core_dev *mdev = dev->mdev;
1418 
1419 	switch (mlx5_get_vport_access_method(ibdev)) {
1420 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1421 		return mlx5_query_mad_ifc_gids(ibdev, port, index, gid);
1422 
1423 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1424 		return mlx5_query_hca_vport_gid(mdev, 0, port, 0, index, gid);
1425 
1426 	default:
1427 		return -EINVAL;
1428 	}
1429 
1430 }
1431 
mlx5_query_hca_nic_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1432 static int mlx5_query_hca_nic_pkey(struct ib_device *ibdev, u32 port,
1433 				   u16 index, u16 *pkey)
1434 {
1435 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1436 	struct mlx5_core_dev *mdev;
1437 	bool put_mdev = true;
1438 	u32 mdev_port_num;
1439 	int err;
1440 
1441 	mdev = mlx5_ib_get_native_port_mdev(dev, port, &mdev_port_num);
1442 	if (!mdev) {
1443 		/* The port isn't affiliated yet, get the PKey from the master
1444 		 * port. For RoCE the PKey tables will be the same.
1445 		 */
1446 		put_mdev = false;
1447 		mdev = dev->mdev;
1448 		mdev_port_num = 1;
1449 	}
1450 
1451 	err = mlx5_query_hca_vport_pkey(mdev, 0, mdev_port_num, 0,
1452 					index, pkey);
1453 	if (put_mdev)
1454 		mlx5_ib_put_native_port_mdev(dev, port);
1455 
1456 	return err;
1457 }
1458 
mlx5_ib_query_pkey(struct ib_device * ibdev,u32 port,u16 index,u16 * pkey)1459 static int mlx5_ib_query_pkey(struct ib_device *ibdev, u32 port, u16 index,
1460 			      u16 *pkey)
1461 {
1462 	switch (mlx5_get_vport_access_method(ibdev)) {
1463 	case MLX5_VPORT_ACCESS_METHOD_MAD:
1464 		return mlx5_query_mad_ifc_pkey(ibdev, port, index, pkey);
1465 
1466 	case MLX5_VPORT_ACCESS_METHOD_HCA:
1467 	case MLX5_VPORT_ACCESS_METHOD_NIC:
1468 		return mlx5_query_hca_nic_pkey(ibdev, port, index, pkey);
1469 	default:
1470 		return -EINVAL;
1471 	}
1472 }
1473 
mlx5_ib_modify_device(struct ib_device * ibdev,int mask,struct ib_device_modify * props)1474 static int mlx5_ib_modify_device(struct ib_device *ibdev, int mask,
1475 				 struct ib_device_modify *props)
1476 {
1477 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1478 	struct mlx5_reg_node_desc in;
1479 	struct mlx5_reg_node_desc out;
1480 	int err;
1481 
1482 	if (mask & ~IB_DEVICE_MODIFY_NODE_DESC)
1483 		return -EOPNOTSUPP;
1484 
1485 	if (!(mask & IB_DEVICE_MODIFY_NODE_DESC))
1486 		return 0;
1487 
1488 	/*
1489 	 * If possible, pass node desc to FW, so it can generate
1490 	 * a 144 trap.  If cmd fails, just ignore.
1491 	 */
1492 	memcpy(&in, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1493 	err = mlx5_core_access_reg(dev->mdev, &in, sizeof(in), &out,
1494 				   sizeof(out), MLX5_REG_NODE_DESC, 0, 1);
1495 	if (err)
1496 		return err;
1497 
1498 	memcpy(ibdev->node_desc, props->node_desc, IB_DEVICE_NODE_DESC_MAX);
1499 
1500 	return err;
1501 }
1502 
set_port_caps_atomic(struct mlx5_ib_dev * dev,u32 port_num,u32 mask,u32 value)1503 static int set_port_caps_atomic(struct mlx5_ib_dev *dev, u32 port_num, u32 mask,
1504 				u32 value)
1505 {
1506 	struct mlx5_hca_vport_context ctx = {};
1507 	struct mlx5_core_dev *mdev;
1508 	u32 mdev_port_num;
1509 	int err;
1510 
1511 	mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num);
1512 	if (!mdev)
1513 		return -ENODEV;
1514 
1515 	err = mlx5_query_hca_vport_context(mdev, 0, mdev_port_num, 0, &ctx);
1516 	if (err)
1517 		goto out;
1518 
1519 	if (~ctx.cap_mask1_perm & mask) {
1520 		mlx5_ib_warn(dev, "trying to change bitmask 0x%X but change supported 0x%X\n",
1521 			     mask, ctx.cap_mask1_perm);
1522 		err = -EINVAL;
1523 		goto out;
1524 	}
1525 
1526 	ctx.cap_mask1 = value;
1527 	ctx.cap_mask1_perm = mask;
1528 	err = mlx5_core_modify_hca_vport_context(mdev, 0, mdev_port_num,
1529 						 0, &ctx);
1530 
1531 out:
1532 	mlx5_ib_put_native_port_mdev(dev, port_num);
1533 
1534 	return err;
1535 }
1536 
mlx5_ib_modify_port(struct ib_device * ibdev,u32 port,int mask,struct ib_port_modify * props)1537 static int mlx5_ib_modify_port(struct ib_device *ibdev, u32 port, int mask,
1538 			       struct ib_port_modify *props)
1539 {
1540 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1541 	struct ib_port_attr attr;
1542 	u32 tmp;
1543 	int err;
1544 	u32 change_mask;
1545 	u32 value;
1546 	bool is_ib = (mlx5_ib_port_link_layer(ibdev, port) ==
1547 		      IB_LINK_LAYER_INFINIBAND);
1548 
1549 	/* CM layer calls ib_modify_port() regardless of the link layer. For
1550 	 * Ethernet ports, qkey violation and Port capabilities are meaningless.
1551 	 */
1552 	if (!is_ib)
1553 		return 0;
1554 
1555 	if (MLX5_CAP_GEN(dev->mdev, ib_virt) && is_ib) {
1556 		change_mask = props->clr_port_cap_mask | props->set_port_cap_mask;
1557 		value = ~props->clr_port_cap_mask | props->set_port_cap_mask;
1558 		return set_port_caps_atomic(dev, port, change_mask, value);
1559 	}
1560 
1561 	mutex_lock(&dev->cap_mask_mutex);
1562 
1563 	err = ib_query_port(ibdev, port, &attr);
1564 	if (err)
1565 		goto out;
1566 
1567 	tmp = (attr.port_cap_flags | props->set_port_cap_mask) &
1568 		~props->clr_port_cap_mask;
1569 
1570 	err = mlx5_set_port_caps(dev->mdev, port, tmp);
1571 
1572 out:
1573 	mutex_unlock(&dev->cap_mask_mutex);
1574 	return err;
1575 }
1576 
print_lib_caps(struct mlx5_ib_dev * dev,u64 caps)1577 static void print_lib_caps(struct mlx5_ib_dev *dev, u64 caps)
1578 {
1579 	mlx5_ib_dbg(dev, "MLX5_LIB_CAP_4K_UAR = %s\n",
1580 		    caps & MLX5_LIB_CAP_4K_UAR ? "y" : "n");
1581 }
1582 
calc_dynamic_bfregs(int uars_per_sys_page)1583 static u16 calc_dynamic_bfregs(int uars_per_sys_page)
1584 {
1585 	/* Large page with non 4k uar support might limit the dynamic size */
1586 	if (uars_per_sys_page == 1  && PAGE_SIZE > 4096)
1587 		return MLX5_MIN_DYN_BFREGS;
1588 
1589 	return MLX5_MAX_DYN_BFREGS;
1590 }
1591 
calc_total_bfregs(struct mlx5_ib_dev * dev,bool lib_uar_4k,struct mlx5_ib_alloc_ucontext_req_v2 * req,struct mlx5_bfreg_info * bfregi)1592 static int calc_total_bfregs(struct mlx5_ib_dev *dev, bool lib_uar_4k,
1593 			     struct mlx5_ib_alloc_ucontext_req_v2 *req,
1594 			     struct mlx5_bfreg_info *bfregi)
1595 {
1596 	int uars_per_sys_page;
1597 	int bfregs_per_sys_page;
1598 	int ref_bfregs = req->total_num_bfregs;
1599 
1600 	if (req->total_num_bfregs == 0)
1601 		return -EINVAL;
1602 
1603 	BUILD_BUG_ON(MLX5_MAX_BFREGS % MLX5_NON_FP_BFREGS_IN_PAGE);
1604 	BUILD_BUG_ON(MLX5_MAX_BFREGS < MLX5_NON_FP_BFREGS_IN_PAGE);
1605 
1606 	if (req->total_num_bfregs > MLX5_MAX_BFREGS)
1607 		return -ENOMEM;
1608 
1609 	uars_per_sys_page = get_uars_per_sys_page(dev, lib_uar_4k);
1610 	bfregs_per_sys_page = uars_per_sys_page * MLX5_NON_FP_BFREGS_PER_UAR;
1611 	/* This holds the required static allocation asked by the user */
1612 	req->total_num_bfregs = ALIGN(req->total_num_bfregs, bfregs_per_sys_page);
1613 	if (req->num_low_latency_bfregs > req->total_num_bfregs - 1)
1614 		return -EINVAL;
1615 
1616 	bfregi->num_static_sys_pages = req->total_num_bfregs / bfregs_per_sys_page;
1617 	bfregi->num_dyn_bfregs = ALIGN(calc_dynamic_bfregs(uars_per_sys_page), bfregs_per_sys_page);
1618 	bfregi->total_num_bfregs = req->total_num_bfregs + bfregi->num_dyn_bfregs;
1619 	bfregi->num_sys_pages = bfregi->total_num_bfregs / bfregs_per_sys_page;
1620 
1621 	mlx5_ib_dbg(dev, "uar_4k: fw support %s, lib support %s, user requested %d bfregs, allocated %d, total bfregs %d, using %d sys pages\n",
1622 		    MLX5_CAP_GEN(dev->mdev, uar_4k) ? "yes" : "no",
1623 		    lib_uar_4k ? "yes" : "no", ref_bfregs,
1624 		    req->total_num_bfregs, bfregi->total_num_bfregs,
1625 		    bfregi->num_sys_pages);
1626 
1627 	return 0;
1628 }
1629 
allocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1630 static int allocate_uars(struct mlx5_ib_dev *dev, struct mlx5_ib_ucontext *context)
1631 {
1632 	struct mlx5_bfreg_info *bfregi;
1633 	int err;
1634 	int i;
1635 
1636 	bfregi = &context->bfregi;
1637 	for (i = 0; i < bfregi->num_static_sys_pages; i++) {
1638 		err = mlx5_cmd_uar_alloc(dev->mdev, &bfregi->sys_pages[i],
1639 					 context->devx_uid);
1640 		if (err)
1641 			goto error;
1642 
1643 		mlx5_ib_dbg(dev, "allocated uar %d\n", bfregi->sys_pages[i]);
1644 	}
1645 
1646 	for (i = bfregi->num_static_sys_pages; i < bfregi->num_sys_pages; i++)
1647 		bfregi->sys_pages[i] = MLX5_IB_INVALID_UAR_INDEX;
1648 
1649 	return 0;
1650 
1651 error:
1652 	for (--i; i >= 0; i--)
1653 		if (mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1654 					 context->devx_uid))
1655 			mlx5_ib_warn(dev, "failed to free uar %d\n", i);
1656 
1657 	return err;
1658 }
1659 
deallocate_uars(struct mlx5_ib_dev * dev,struct mlx5_ib_ucontext * context)1660 static void deallocate_uars(struct mlx5_ib_dev *dev,
1661 			    struct mlx5_ib_ucontext *context)
1662 {
1663 	struct mlx5_bfreg_info *bfregi;
1664 	int i;
1665 
1666 	bfregi = &context->bfregi;
1667 	for (i = 0; i < bfregi->num_sys_pages; i++)
1668 		if (i < bfregi->num_static_sys_pages ||
1669 		    bfregi->sys_pages[i] != MLX5_IB_INVALID_UAR_INDEX)
1670 			mlx5_cmd_uar_dealloc(dev->mdev, bfregi->sys_pages[i],
1671 					     context->devx_uid);
1672 }
1673 
mlx5_ib_enable_lb(struct mlx5_ib_dev * dev,bool td,bool qp)1674 int mlx5_ib_enable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1675 {
1676 	int err = 0;
1677 
1678 	mutex_lock(&dev->lb.mutex);
1679 	if (td)
1680 		dev->lb.user_td++;
1681 	if (qp)
1682 		dev->lb.qps++;
1683 
1684 	if (dev->lb.user_td == 2 ||
1685 	    dev->lb.qps == 1) {
1686 		if (!dev->lb.enabled) {
1687 			err = mlx5_nic_vport_update_local_lb(dev->mdev, true);
1688 			dev->lb.enabled = true;
1689 		}
1690 	}
1691 
1692 	mutex_unlock(&dev->lb.mutex);
1693 
1694 	return err;
1695 }
1696 
mlx5_ib_disable_lb(struct mlx5_ib_dev * dev,bool td,bool qp)1697 void mlx5_ib_disable_lb(struct mlx5_ib_dev *dev, bool td, bool qp)
1698 {
1699 	mutex_lock(&dev->lb.mutex);
1700 	if (td)
1701 		dev->lb.user_td--;
1702 	if (qp)
1703 		dev->lb.qps--;
1704 
1705 	if (dev->lb.user_td == 1 &&
1706 	    dev->lb.qps == 0) {
1707 		if (dev->lb.enabled) {
1708 			mlx5_nic_vport_update_local_lb(dev->mdev, false);
1709 			dev->lb.enabled = false;
1710 		}
1711 	}
1712 
1713 	mutex_unlock(&dev->lb.mutex);
1714 }
1715 
mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev * dev,u32 * tdn,u16 uid)1716 static int mlx5_ib_alloc_transport_domain(struct mlx5_ib_dev *dev, u32 *tdn,
1717 					  u16 uid)
1718 {
1719 	int err;
1720 
1721 	if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1722 		return 0;
1723 
1724 	err = mlx5_cmd_alloc_transport_domain(dev->mdev, tdn, uid);
1725 	if (err)
1726 		return err;
1727 
1728 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1729 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1730 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1731 		return err;
1732 
1733 	return mlx5_ib_enable_lb(dev, true, false);
1734 }
1735 
mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev * dev,u32 tdn,u16 uid)1736 static void mlx5_ib_dealloc_transport_domain(struct mlx5_ib_dev *dev, u32 tdn,
1737 					     u16 uid)
1738 {
1739 	if (!MLX5_CAP_GEN(dev->mdev, log_max_transport_domain))
1740 		return;
1741 
1742 	mlx5_cmd_dealloc_transport_domain(dev->mdev, tdn, uid);
1743 
1744 	if ((MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) ||
1745 	    (!MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) &&
1746 	     !MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
1747 		return;
1748 
1749 	mlx5_ib_disable_lb(dev, true, false);
1750 }
1751 
set_ucontext_resp(struct ib_ucontext * uctx,struct mlx5_ib_alloc_ucontext_resp * resp)1752 static int set_ucontext_resp(struct ib_ucontext *uctx,
1753 			     struct mlx5_ib_alloc_ucontext_resp *resp)
1754 {
1755 	struct ib_device *ibdev = uctx->device;
1756 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1757 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1758 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
1759 	int err;
1760 
1761 	if (MLX5_CAP_GEN(dev->mdev, dump_fill_mkey)) {
1762 		err = mlx5_cmd_dump_fill_mkey(dev->mdev,
1763 					      &resp->dump_fill_mkey);
1764 		if (err)
1765 			return err;
1766 		resp->comp_mask |=
1767 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_DUMP_FILL_MKEY;
1768 	}
1769 
1770 	resp->qp_tab_size = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp);
1771 	if (dev->wc_support)
1772 		resp->bf_reg_size = 1 << MLX5_CAP_GEN(dev->mdev,
1773 						      log_bf_reg_size);
1774 	resp->cache_line_size = cache_line_size();
1775 	resp->max_sq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq);
1776 	resp->max_rq_desc_sz = MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq);
1777 	resp->max_send_wqebb = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1778 	resp->max_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz);
1779 	resp->max_srq_recv_wr = 1 << MLX5_CAP_GEN(dev->mdev, log_max_srq_sz);
1780 	resp->cqe_version = context->cqe_version;
1781 	resp->log_uar_size = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1782 				MLX5_ADAPTER_PAGE_SHIFT : PAGE_SHIFT;
1783 	resp->num_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
1784 					MLX5_CAP_GEN(dev->mdev,
1785 						     num_of_uars_per_page) : 1;
1786 	resp->tot_bfregs = bfregi->lib_uar_dyn ? 0 :
1787 			bfregi->total_num_bfregs - bfregi->num_dyn_bfregs;
1788 	resp->num_ports = dev->num_ports;
1789 	resp->cmds_supp_uhw |= MLX5_USER_CMDS_SUPP_UHW_QUERY_DEVICE |
1790 				      MLX5_USER_CMDS_SUPP_UHW_CREATE_AH;
1791 
1792 	if (mlx5_ib_port_link_layer(ibdev, 1) == IB_LINK_LAYER_ETHERNET) {
1793 		mlx5_query_min_inline(dev->mdev, &resp->eth_min_inline);
1794 		resp->eth_min_inline++;
1795 	}
1796 
1797 	if (dev->mdev->clock_info)
1798 		resp->clock_info_versions = BIT(MLX5_IB_CLOCK_INFO_V1);
1799 
1800 	/*
1801 	 * We don't want to expose information from the PCI bar that is located
1802 	 * after 4096 bytes, so if the arch only supports larger pages, let's
1803 	 * pretend we don't support reading the HCA's core clock. This is also
1804 	 * forced by mmap function.
1805 	 */
1806 	if (PAGE_SIZE <= 4096) {
1807 		resp->comp_mask |=
1808 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_CORE_CLOCK_OFFSET;
1809 		resp->hca_core_clock_offset =
1810 			offsetof(struct mlx5_init_seg,
1811 				 internal_timer_h) % PAGE_SIZE;
1812 	}
1813 
1814 	if (MLX5_CAP_GEN(dev->mdev, ece_support))
1815 		resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_ECE;
1816 
1817 	if (rt_supported(MLX5_CAP_GEN(dev->mdev, sq_ts_format)) &&
1818 	    rt_supported(MLX5_CAP_GEN(dev->mdev, rq_ts_format)) &&
1819 	    rt_supported(MLX5_CAP_ROCE(dev->mdev, qp_ts_format)))
1820 		resp->comp_mask |=
1821 			MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_REAL_TIME_TS;
1822 
1823 	resp->num_dyn_bfregs = bfregi->num_dyn_bfregs;
1824 
1825 	if (MLX5_CAP_GEN(dev->mdev, drain_sigerr))
1826 		resp->comp_mask |= MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_SQD2RTS;
1827 
1828 	resp->comp_mask |=
1829 		MLX5_IB_ALLOC_UCONTEXT_RESP_MASK_MKEY_UPDATE_TAG;
1830 
1831 	return 0;
1832 }
1833 
mlx5_ib_alloc_ucontext(struct ib_ucontext * uctx,struct ib_udata * udata)1834 static int mlx5_ib_alloc_ucontext(struct ib_ucontext *uctx,
1835 				  struct ib_udata *udata)
1836 {
1837 	struct ib_device *ibdev = uctx->device;
1838 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
1839 	struct mlx5_ib_alloc_ucontext_req_v2 req = {};
1840 	struct mlx5_ib_alloc_ucontext_resp resp = {};
1841 	struct mlx5_ib_ucontext *context = to_mucontext(uctx);
1842 	struct mlx5_bfreg_info *bfregi;
1843 	int ver;
1844 	int err;
1845 	size_t min_req_v2 = offsetof(struct mlx5_ib_alloc_ucontext_req_v2,
1846 				     max_cqe_version);
1847 	bool lib_uar_4k;
1848 	bool lib_uar_dyn;
1849 
1850 	if (!dev->ib_active)
1851 		return -EAGAIN;
1852 
1853 	if (udata->inlen == sizeof(struct mlx5_ib_alloc_ucontext_req))
1854 		ver = 0;
1855 	else if (udata->inlen >= min_req_v2)
1856 		ver = 2;
1857 	else
1858 		return -EINVAL;
1859 
1860 	err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req)));
1861 	if (err)
1862 		return err;
1863 
1864 	if (req.flags & ~MLX5_IB_ALLOC_UCTX_DEVX)
1865 		return -EOPNOTSUPP;
1866 
1867 	if (req.comp_mask || req.reserved0 || req.reserved1 || req.reserved2)
1868 		return -EOPNOTSUPP;
1869 
1870 	req.total_num_bfregs = ALIGN(req.total_num_bfregs,
1871 				    MLX5_NON_FP_BFREGS_PER_UAR);
1872 	if (req.num_low_latency_bfregs > req.total_num_bfregs - 1)
1873 		return -EINVAL;
1874 
1875 	if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX) {
1876 		err = mlx5_ib_devx_create(dev, true);
1877 		if (err < 0)
1878 			goto out_ctx;
1879 		context->devx_uid = err;
1880 	}
1881 
1882 	lib_uar_4k = req.lib_caps & MLX5_LIB_CAP_4K_UAR;
1883 	lib_uar_dyn = req.lib_caps & MLX5_LIB_CAP_DYN_UAR;
1884 	bfregi = &context->bfregi;
1885 
1886 	if (lib_uar_dyn) {
1887 		bfregi->lib_uar_dyn = lib_uar_dyn;
1888 		goto uar_done;
1889 	}
1890 
1891 	/* updates req->total_num_bfregs */
1892 	err = calc_total_bfregs(dev, lib_uar_4k, &req, bfregi);
1893 	if (err)
1894 		goto out_devx;
1895 
1896 	mutex_init(&bfregi->lock);
1897 	bfregi->lib_uar_4k = lib_uar_4k;
1898 	bfregi->count = kcalloc(bfregi->total_num_bfregs, sizeof(*bfregi->count),
1899 				GFP_KERNEL);
1900 	if (!bfregi->count) {
1901 		err = -ENOMEM;
1902 		goto out_devx;
1903 	}
1904 
1905 	bfregi->sys_pages = kcalloc(bfregi->num_sys_pages,
1906 				    sizeof(*bfregi->sys_pages),
1907 				    GFP_KERNEL);
1908 	if (!bfregi->sys_pages) {
1909 		err = -ENOMEM;
1910 		goto out_count;
1911 	}
1912 
1913 	err = allocate_uars(dev, context);
1914 	if (err)
1915 		goto out_sys_pages;
1916 
1917 uar_done:
1918 	err = mlx5_ib_alloc_transport_domain(dev, &context->tdn,
1919 					     context->devx_uid);
1920 	if (err)
1921 		goto out_uars;
1922 
1923 	INIT_LIST_HEAD(&context->db_page_list);
1924 	mutex_init(&context->db_page_mutex);
1925 
1926 	context->cqe_version = min_t(__u8,
1927 				 (__u8)MLX5_CAP_GEN(dev->mdev, cqe_version),
1928 				 req.max_cqe_version);
1929 
1930 	err = set_ucontext_resp(uctx, &resp);
1931 	if (err)
1932 		goto out_mdev;
1933 
1934 	resp.response_length = min(udata->outlen, sizeof(resp));
1935 	err = ib_copy_to_udata(udata, &resp, resp.response_length);
1936 	if (err)
1937 		goto out_mdev;
1938 
1939 	bfregi->ver = ver;
1940 	bfregi->num_low_latency_bfregs = req.num_low_latency_bfregs;
1941 	context->lib_caps = req.lib_caps;
1942 	print_lib_caps(dev, context->lib_caps);
1943 
1944 	if (mlx5_ib_lag_should_assign_affinity(dev)) {
1945 		u32 port = mlx5_core_native_port_num(dev->mdev) - 1;
1946 
1947 		atomic_set(&context->tx_port_affinity,
1948 			   atomic_add_return(
1949 				   1, &dev->port[port].roce.tx_port_affinity));
1950 	}
1951 
1952 	return 0;
1953 
1954 out_mdev:
1955 	mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
1956 
1957 out_uars:
1958 	deallocate_uars(dev, context);
1959 
1960 out_sys_pages:
1961 	kfree(bfregi->sys_pages);
1962 
1963 out_count:
1964 	kfree(bfregi->count);
1965 
1966 out_devx:
1967 	if (req.flags & MLX5_IB_ALLOC_UCTX_DEVX)
1968 		mlx5_ib_devx_destroy(dev, context->devx_uid);
1969 
1970 out_ctx:
1971 	return err;
1972 }
1973 
mlx5_ib_query_ucontext(struct ib_ucontext * ibcontext,struct uverbs_attr_bundle * attrs)1974 static int mlx5_ib_query_ucontext(struct ib_ucontext *ibcontext,
1975 				  struct uverbs_attr_bundle *attrs)
1976 {
1977 	struct mlx5_ib_alloc_ucontext_resp uctx_resp = {};
1978 	int ret;
1979 
1980 	ret = set_ucontext_resp(ibcontext, &uctx_resp);
1981 	if (ret)
1982 		return ret;
1983 
1984 	uctx_resp.response_length =
1985 		min_t(size_t,
1986 		      uverbs_attr_get_len(attrs,
1987 				MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX),
1988 		      sizeof(uctx_resp));
1989 
1990 	ret = uverbs_copy_to_struct_or_zero(attrs,
1991 					MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
1992 					&uctx_resp,
1993 					sizeof(uctx_resp));
1994 	return ret;
1995 }
1996 
mlx5_ib_dealloc_ucontext(struct ib_ucontext * ibcontext)1997 static void mlx5_ib_dealloc_ucontext(struct ib_ucontext *ibcontext)
1998 {
1999 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2000 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2001 	struct mlx5_bfreg_info *bfregi;
2002 
2003 	bfregi = &context->bfregi;
2004 	mlx5_ib_dealloc_transport_domain(dev, context->tdn, context->devx_uid);
2005 
2006 	deallocate_uars(dev, context);
2007 	kfree(bfregi->sys_pages);
2008 	kfree(bfregi->count);
2009 
2010 	if (context->devx_uid)
2011 		mlx5_ib_devx_destroy(dev, context->devx_uid);
2012 }
2013 
uar_index2pfn(struct mlx5_ib_dev * dev,int uar_idx)2014 static phys_addr_t uar_index2pfn(struct mlx5_ib_dev *dev,
2015 				 int uar_idx)
2016 {
2017 	int fw_uars_per_page;
2018 
2019 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ? MLX5_UARS_IN_PAGE : 1;
2020 
2021 	return (dev->mdev->bar_addr >> PAGE_SHIFT) + uar_idx / fw_uars_per_page;
2022 }
2023 
uar_index2paddress(struct mlx5_ib_dev * dev,int uar_idx)2024 static u64 uar_index2paddress(struct mlx5_ib_dev *dev,
2025 				 int uar_idx)
2026 {
2027 	unsigned int fw_uars_per_page;
2028 
2029 	fw_uars_per_page = MLX5_CAP_GEN(dev->mdev, uar_4k) ?
2030 				MLX5_UARS_IN_PAGE : 1;
2031 
2032 	return (dev->mdev->bar_addr + (uar_idx / fw_uars_per_page) * PAGE_SIZE);
2033 }
2034 
get_command(unsigned long offset)2035 static int get_command(unsigned long offset)
2036 {
2037 	return (offset >> MLX5_IB_MMAP_CMD_SHIFT) & MLX5_IB_MMAP_CMD_MASK;
2038 }
2039 
get_arg(unsigned long offset)2040 static int get_arg(unsigned long offset)
2041 {
2042 	return offset & ((1 << MLX5_IB_MMAP_CMD_SHIFT) - 1);
2043 }
2044 
get_index(unsigned long offset)2045 static int get_index(unsigned long offset)
2046 {
2047 	return get_arg(offset);
2048 }
2049 
2050 /* Index resides in an extra byte to enable larger values than 255 */
get_extended_index(unsigned long offset)2051 static int get_extended_index(unsigned long offset)
2052 {
2053 	return get_arg(offset) | ((offset >> 16) & 0xff) << 8;
2054 }
2055 
2056 
mlx5_ib_disassociate_ucontext(struct ib_ucontext * ibcontext)2057 static void mlx5_ib_disassociate_ucontext(struct ib_ucontext *ibcontext)
2058 {
2059 }
2060 
mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)2061 static inline char *mmap_cmd2str(enum mlx5_ib_mmap_cmd cmd)
2062 {
2063 	switch (cmd) {
2064 	case MLX5_IB_MMAP_WC_PAGE:
2065 		return "WC";
2066 	case MLX5_IB_MMAP_REGULAR_PAGE:
2067 		return "best effort WC";
2068 	case MLX5_IB_MMAP_NC_PAGE:
2069 		return "NC";
2070 	case MLX5_IB_MMAP_DEVICE_MEM:
2071 		return "Device Memory";
2072 	default:
2073 		return NULL;
2074 	}
2075 }
2076 
mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)2077 static int mlx5_ib_mmap_clock_info_page(struct mlx5_ib_dev *dev,
2078 					struct vm_area_struct *vma,
2079 					struct mlx5_ib_ucontext *context)
2080 {
2081 	if ((vma->vm_end - vma->vm_start != PAGE_SIZE) ||
2082 	    !(vma->vm_flags & VM_SHARED))
2083 		return -EINVAL;
2084 
2085 	if (get_index(vma->vm_pgoff) != MLX5_IB_CLOCK_INFO_V1)
2086 		return -EOPNOTSUPP;
2087 
2088 	if (vma->vm_flags & (VM_WRITE | VM_EXEC))
2089 		return -EPERM;
2090 	vma->vm_flags &= ~VM_MAYWRITE;
2091 
2092 	if (!dev->mdev->clock_info)
2093 		return -EOPNOTSUPP;
2094 
2095 	return vm_insert_page(vma, vma->vm_start,
2096 			      virt_to_page(dev->mdev->clock_info));
2097 }
2098 
mlx5_ib_mmap_free(struct rdma_user_mmap_entry * entry)2099 static void mlx5_ib_mmap_free(struct rdma_user_mmap_entry *entry)
2100 {
2101 	struct mlx5_user_mmap_entry *mentry = to_mmmap(entry);
2102 	struct mlx5_ib_dev *dev = to_mdev(entry->ucontext->device);
2103 	struct mlx5_var_table *var_table = &dev->var_table;
2104 	struct mlx5_ib_ucontext *context = to_mucontext(entry->ucontext);
2105 
2106 	switch (mentry->mmap_flag) {
2107 	case MLX5_IB_MMAP_TYPE_MEMIC:
2108 	case MLX5_IB_MMAP_TYPE_MEMIC_OP:
2109 		mlx5_ib_dm_mmap_free(dev, mentry);
2110 		break;
2111 	case MLX5_IB_MMAP_TYPE_VAR:
2112 		mutex_lock(&var_table->bitmap_lock);
2113 		clear_bit(mentry->page_idx, var_table->bitmap);
2114 		mutex_unlock(&var_table->bitmap_lock);
2115 		kfree(mentry);
2116 		break;
2117 	case MLX5_IB_MMAP_TYPE_UAR_WC:
2118 	case MLX5_IB_MMAP_TYPE_UAR_NC:
2119 		mlx5_cmd_uar_dealloc(dev->mdev, mentry->page_idx,
2120 				     context->devx_uid);
2121 		kfree(mentry);
2122 		break;
2123 	default:
2124 		WARN_ON(true);
2125 	}
2126 }
2127 
uar_mmap(struct mlx5_ib_dev * dev,enum mlx5_ib_mmap_cmd cmd,struct vm_area_struct * vma,struct mlx5_ib_ucontext * context)2128 static int uar_mmap(struct mlx5_ib_dev *dev, enum mlx5_ib_mmap_cmd cmd,
2129 		    struct vm_area_struct *vma,
2130 		    struct mlx5_ib_ucontext *context)
2131 {
2132 	struct mlx5_bfreg_info *bfregi = &context->bfregi;
2133 	int err;
2134 	unsigned long idx;
2135 	phys_addr_t pfn;
2136 	pgprot_t prot;
2137 	u32 bfreg_dyn_idx = 0;
2138 	u32 uar_index;
2139 	int dyn_uar = (cmd == MLX5_IB_MMAP_ALLOC_WC);
2140 	int max_valid_idx = dyn_uar ? bfregi->num_sys_pages :
2141 				bfregi->num_static_sys_pages;
2142 
2143 	if (bfregi->lib_uar_dyn)
2144 		return -EINVAL;
2145 
2146 	if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2147 		return -EINVAL;
2148 
2149 	if (dyn_uar)
2150 		idx = get_extended_index(vma->vm_pgoff) + bfregi->num_static_sys_pages;
2151 	else
2152 		idx = get_index(vma->vm_pgoff);
2153 
2154 	if (idx >= max_valid_idx) {
2155 		mlx5_ib_warn(dev, "invalid uar index %lu, max=%d\n",
2156 			     idx, max_valid_idx);
2157 		return -EINVAL;
2158 	}
2159 
2160 	switch (cmd) {
2161 	case MLX5_IB_MMAP_WC_PAGE:
2162 	case MLX5_IB_MMAP_ALLOC_WC:
2163 	case MLX5_IB_MMAP_REGULAR_PAGE:
2164 		/* For MLX5_IB_MMAP_REGULAR_PAGE do the best effort to get WC */
2165 		prot = pgprot_writecombine(vma->vm_page_prot);
2166 		break;
2167 	case MLX5_IB_MMAP_NC_PAGE:
2168 		prot = pgprot_noncached(vma->vm_page_prot);
2169 		break;
2170 	default:
2171 		return -EINVAL;
2172 	}
2173 
2174 	if (dyn_uar) {
2175 		int uars_per_page;
2176 
2177 		uars_per_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k);
2178 		bfreg_dyn_idx = idx * (uars_per_page * MLX5_NON_FP_BFREGS_PER_UAR);
2179 		if (bfreg_dyn_idx >= bfregi->total_num_bfregs) {
2180 			mlx5_ib_warn(dev, "invalid bfreg_dyn_idx %u, max=%u\n",
2181 				     bfreg_dyn_idx, bfregi->total_num_bfregs);
2182 			return -EINVAL;
2183 		}
2184 
2185 		mutex_lock(&bfregi->lock);
2186 		/* Fail if uar already allocated, first bfreg index of each
2187 		 * page holds its count.
2188 		 */
2189 		if (bfregi->count[bfreg_dyn_idx]) {
2190 			mlx5_ib_warn(dev, "wrong offset, idx %lu is busy, bfregn=%u\n", idx, bfreg_dyn_idx);
2191 			mutex_unlock(&bfregi->lock);
2192 			return -EINVAL;
2193 		}
2194 
2195 		bfregi->count[bfreg_dyn_idx]++;
2196 		mutex_unlock(&bfregi->lock);
2197 
2198 		err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index,
2199 					 context->devx_uid);
2200 		if (err) {
2201 			mlx5_ib_warn(dev, "UAR alloc failed\n");
2202 			goto free_bfreg;
2203 		}
2204 	} else {
2205 		uar_index = bfregi->sys_pages[idx];
2206 	}
2207 
2208 	pfn = uar_index2pfn(dev, uar_index);
2209 	mlx5_ib_dbg(dev, "uar idx 0x%lx, pfn %pa\n", idx, &pfn);
2210 
2211 	err = rdma_user_mmap_io(&context->ibucontext, vma, pfn, PAGE_SIZE,
2212 				prot, NULL);
2213 	if (err) {
2214 		mlx5_ib_err(dev,
2215 			    "rdma_user_mmap_io failed with error=%d, mmap_cmd=%s\n",
2216 			    err, mmap_cmd2str(cmd));
2217 		goto err;
2218 	}
2219 
2220 	if (dyn_uar)
2221 		bfregi->sys_pages[idx] = uar_index;
2222 	return 0;
2223 
2224 err:
2225 	if (!dyn_uar)
2226 		return err;
2227 
2228 	mlx5_cmd_uar_dealloc(dev->mdev, idx, context->devx_uid);
2229 
2230 free_bfreg:
2231 	mlx5_ib_free_bfreg(dev, bfregi, bfreg_dyn_idx);
2232 
2233 	return err;
2234 }
2235 
mlx5_vma_to_pgoff(struct vm_area_struct * vma)2236 static unsigned long mlx5_vma_to_pgoff(struct vm_area_struct *vma)
2237 {
2238 	unsigned long idx;
2239 	u8 command;
2240 
2241 	command = get_command(vma->vm_pgoff);
2242 	idx = get_extended_index(vma->vm_pgoff);
2243 
2244 	return (command << 16 | idx);
2245 }
2246 
mlx5_ib_mmap_offset(struct mlx5_ib_dev * dev,struct vm_area_struct * vma,struct ib_ucontext * ucontext)2247 static int mlx5_ib_mmap_offset(struct mlx5_ib_dev *dev,
2248 			       struct vm_area_struct *vma,
2249 			       struct ib_ucontext *ucontext)
2250 {
2251 	struct mlx5_user_mmap_entry *mentry;
2252 	struct rdma_user_mmap_entry *entry;
2253 	unsigned long pgoff;
2254 	pgprot_t prot;
2255 	phys_addr_t pfn;
2256 	int ret;
2257 
2258 	pgoff = mlx5_vma_to_pgoff(vma);
2259 	entry = rdma_user_mmap_entry_get_pgoff(ucontext, pgoff);
2260 	if (!entry)
2261 		return -EINVAL;
2262 
2263 	mentry = to_mmmap(entry);
2264 	pfn = (mentry->address >> PAGE_SHIFT);
2265 	if (mentry->mmap_flag == MLX5_IB_MMAP_TYPE_VAR ||
2266 	    mentry->mmap_flag == MLX5_IB_MMAP_TYPE_UAR_NC)
2267 		prot = pgprot_noncached(vma->vm_page_prot);
2268 	else
2269 		prot = pgprot_writecombine(vma->vm_page_prot);
2270 	ret = rdma_user_mmap_io(ucontext, vma, pfn,
2271 				entry->npages * PAGE_SIZE,
2272 				prot,
2273 				entry);
2274 	rdma_user_mmap_entry_put(&mentry->rdma_entry);
2275 	return ret;
2276 }
2277 
mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry * entry)2278 static u64 mlx5_entry_to_mmap_offset(struct mlx5_user_mmap_entry *entry)
2279 {
2280 	u64 cmd = (entry->rdma_entry.start_pgoff >> 16) & 0xFFFF;
2281 	u64 index = entry->rdma_entry.start_pgoff & 0xFFFF;
2282 
2283 	return (((index >> 8) << 16) | (cmd << MLX5_IB_MMAP_CMD_SHIFT) |
2284 		(index & 0xFF)) << PAGE_SHIFT;
2285 }
2286 
mlx5_ib_mmap(struct ib_ucontext * ibcontext,struct vm_area_struct * vma)2287 static int mlx5_ib_mmap(struct ib_ucontext *ibcontext, struct vm_area_struct *vma)
2288 {
2289 	struct mlx5_ib_ucontext *context = to_mucontext(ibcontext);
2290 	struct mlx5_ib_dev *dev = to_mdev(ibcontext->device);
2291 	unsigned long command;
2292 	phys_addr_t pfn;
2293 
2294 	command = get_command(vma->vm_pgoff);
2295 	switch (command) {
2296 	case MLX5_IB_MMAP_WC_PAGE:
2297 	case MLX5_IB_MMAP_ALLOC_WC:
2298 		if (!dev->wc_support)
2299 			return -EPERM;
2300 		fallthrough;
2301 	case MLX5_IB_MMAP_NC_PAGE:
2302 	case MLX5_IB_MMAP_REGULAR_PAGE:
2303 		return uar_mmap(dev, command, vma, context);
2304 
2305 	case MLX5_IB_MMAP_GET_CONTIGUOUS_PAGES:
2306 		return -ENOSYS;
2307 
2308 	case MLX5_IB_MMAP_CORE_CLOCK:
2309 		if (vma->vm_end - vma->vm_start != PAGE_SIZE)
2310 			return -EINVAL;
2311 
2312 		if (vma->vm_flags & VM_WRITE)
2313 			return -EPERM;
2314 		vma->vm_flags &= ~VM_MAYWRITE;
2315 
2316 		/* Don't expose to user-space information it shouldn't have */
2317 		if (PAGE_SIZE > 4096)
2318 			return -EOPNOTSUPP;
2319 
2320 		pfn = (dev->mdev->iseg_base +
2321 		       offsetof(struct mlx5_init_seg, internal_timer_h)) >>
2322 			PAGE_SHIFT;
2323 		return rdma_user_mmap_io(&context->ibucontext, vma, pfn,
2324 					 PAGE_SIZE,
2325 					 pgprot_noncached(vma->vm_page_prot),
2326 					 NULL);
2327 	case MLX5_IB_MMAP_CLOCK_INFO:
2328 		return mlx5_ib_mmap_clock_info_page(dev, vma, context);
2329 
2330 	default:
2331 		return mlx5_ib_mmap_offset(dev, vma, ibcontext);
2332 	}
2333 
2334 	return 0;
2335 }
2336 
mlx5_ib_alloc_pd(struct ib_pd * ibpd,struct ib_udata * udata)2337 static int mlx5_ib_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
2338 {
2339 	struct mlx5_ib_pd *pd = to_mpd(ibpd);
2340 	struct ib_device *ibdev = ibpd->device;
2341 	struct mlx5_ib_alloc_pd_resp resp;
2342 	int err;
2343 	u32 out[MLX5_ST_SZ_DW(alloc_pd_out)] = {};
2344 	u32 in[MLX5_ST_SZ_DW(alloc_pd_in)] = {};
2345 	u16 uid = 0;
2346 	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
2347 		udata, struct mlx5_ib_ucontext, ibucontext);
2348 
2349 	uid = context ? context->devx_uid : 0;
2350 	MLX5_SET(alloc_pd_in, in, opcode, MLX5_CMD_OP_ALLOC_PD);
2351 	MLX5_SET(alloc_pd_in, in, uid, uid);
2352 	err = mlx5_cmd_exec_inout(to_mdev(ibdev)->mdev, alloc_pd, in, out);
2353 	if (err)
2354 		return err;
2355 
2356 	pd->pdn = MLX5_GET(alloc_pd_out, out, pd);
2357 	pd->uid = uid;
2358 	if (udata) {
2359 		resp.pdn = pd->pdn;
2360 		if (ib_copy_to_udata(udata, &resp, sizeof(resp))) {
2361 			mlx5_cmd_dealloc_pd(to_mdev(ibdev)->mdev, pd->pdn, uid);
2362 			return -EFAULT;
2363 		}
2364 	}
2365 
2366 	return 0;
2367 }
2368 
mlx5_ib_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)2369 static int mlx5_ib_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
2370 {
2371 	struct mlx5_ib_dev *mdev = to_mdev(pd->device);
2372 	struct mlx5_ib_pd *mpd = to_mpd(pd);
2373 
2374 	return mlx5_cmd_dealloc_pd(mdev->mdev, mpd->pdn, mpd->uid);
2375 }
2376 
mlx5_ib_mcg_attach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)2377 static int mlx5_ib_mcg_attach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2378 {
2379 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2380 	struct mlx5_ib_qp *mqp = to_mqp(ibqp);
2381 	int err;
2382 	u16 uid;
2383 
2384 	uid = ibqp->pd ?
2385 		to_mpd(ibqp->pd)->uid : 0;
2386 
2387 	if (mqp->flags & IB_QP_CREATE_SOURCE_QPN) {
2388 		mlx5_ib_dbg(dev, "Attaching a multi cast group to underlay QP is not supported\n");
2389 		return -EOPNOTSUPP;
2390 	}
2391 
2392 	err = mlx5_cmd_attach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2393 	if (err)
2394 		mlx5_ib_warn(dev, "failed attaching QPN 0x%x, MGID %pI6\n",
2395 			     ibqp->qp_num, gid->raw);
2396 
2397 	return err;
2398 }
2399 
mlx5_ib_mcg_detach(struct ib_qp * ibqp,union ib_gid * gid,u16 lid)2400 static int mlx5_ib_mcg_detach(struct ib_qp *ibqp, union ib_gid *gid, u16 lid)
2401 {
2402 	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
2403 	int err;
2404 	u16 uid;
2405 
2406 	uid = ibqp->pd ?
2407 		to_mpd(ibqp->pd)->uid : 0;
2408 	err = mlx5_cmd_detach_mcg(dev->mdev, gid, ibqp->qp_num, uid);
2409 	if (err)
2410 		mlx5_ib_warn(dev, "failed detaching QPN 0x%x, MGID %pI6\n",
2411 			     ibqp->qp_num, gid->raw);
2412 
2413 	return err;
2414 }
2415 
init_node_data(struct mlx5_ib_dev * dev)2416 static int init_node_data(struct mlx5_ib_dev *dev)
2417 {
2418 	int err;
2419 
2420 	err = mlx5_query_node_desc(dev, dev->ib_dev.node_desc);
2421 	if (err)
2422 		return err;
2423 
2424 	dev->mdev->rev_id = dev->mdev->pdev->revision;
2425 
2426 	return mlx5_query_node_guid(dev, &dev->ib_dev.node_guid);
2427 }
2428 
fw_pages_show(struct device * device,struct device_attribute * attr,char * buf)2429 static ssize_t fw_pages_show(struct device *device,
2430 			     struct device_attribute *attr, char *buf)
2431 {
2432 	struct mlx5_ib_dev *dev =
2433 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2434 
2435 	return sysfs_emit(buf, "%d\n", dev->mdev->priv.fw_pages);
2436 }
2437 static DEVICE_ATTR_RO(fw_pages);
2438 
reg_pages_show(struct device * device,struct device_attribute * attr,char * buf)2439 static ssize_t reg_pages_show(struct device *device,
2440 			      struct device_attribute *attr, char *buf)
2441 {
2442 	struct mlx5_ib_dev *dev =
2443 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2444 
2445 	return sysfs_emit(buf, "%d\n", atomic_read(&dev->mdev->priv.reg_pages));
2446 }
2447 static DEVICE_ATTR_RO(reg_pages);
2448 
hca_type_show(struct device * device,struct device_attribute * attr,char * buf)2449 static ssize_t hca_type_show(struct device *device,
2450 			     struct device_attribute *attr, char *buf)
2451 {
2452 	struct mlx5_ib_dev *dev =
2453 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2454 
2455 	return sysfs_emit(buf, "MT%d\n", dev->mdev->pdev->device);
2456 }
2457 static DEVICE_ATTR_RO(hca_type);
2458 
hw_rev_show(struct device * device,struct device_attribute * attr,char * buf)2459 static ssize_t hw_rev_show(struct device *device,
2460 			   struct device_attribute *attr, char *buf)
2461 {
2462 	struct mlx5_ib_dev *dev =
2463 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2464 
2465 	return sysfs_emit(buf, "%x\n", dev->mdev->rev_id);
2466 }
2467 static DEVICE_ATTR_RO(hw_rev);
2468 
board_id_show(struct device * device,struct device_attribute * attr,char * buf)2469 static ssize_t board_id_show(struct device *device,
2470 			     struct device_attribute *attr, char *buf)
2471 {
2472 	struct mlx5_ib_dev *dev =
2473 		rdma_device_to_drv_device(device, struct mlx5_ib_dev, ib_dev);
2474 
2475 	return sysfs_emit(buf, "%.*s\n", MLX5_BOARD_ID_LEN,
2476 			  dev->mdev->board_id);
2477 }
2478 static DEVICE_ATTR_RO(board_id);
2479 
2480 static struct attribute *mlx5_class_attributes[] = {
2481 	&dev_attr_hw_rev.attr,
2482 	&dev_attr_hca_type.attr,
2483 	&dev_attr_board_id.attr,
2484 	&dev_attr_fw_pages.attr,
2485 	&dev_attr_reg_pages.attr,
2486 	NULL,
2487 };
2488 
2489 static const struct attribute_group mlx5_attr_group = {
2490 	.attrs = mlx5_class_attributes,
2491 };
2492 
pkey_change_handler(struct work_struct * work)2493 static void pkey_change_handler(struct work_struct *work)
2494 {
2495 	struct mlx5_ib_port_resources *ports =
2496 		container_of(work, struct mlx5_ib_port_resources,
2497 			     pkey_change_work);
2498 
2499 	if (!ports->gsi)
2500 		/*
2501 		 * We got this event before device was fully configured
2502 		 * and MAD registration code wasn't called/finished yet.
2503 		 */
2504 		return;
2505 
2506 	mlx5_ib_gsi_pkey_change(ports->gsi);
2507 }
2508 
mlx5_ib_handle_internal_error(struct mlx5_ib_dev * ibdev)2509 static void mlx5_ib_handle_internal_error(struct mlx5_ib_dev *ibdev)
2510 {
2511 	struct mlx5_ib_qp *mqp;
2512 	struct mlx5_ib_cq *send_mcq, *recv_mcq;
2513 	struct mlx5_core_cq *mcq;
2514 	struct list_head cq_armed_list;
2515 	unsigned long flags_qp;
2516 	unsigned long flags_cq;
2517 	unsigned long flags;
2518 
2519 	INIT_LIST_HEAD(&cq_armed_list);
2520 
2521 	/* Go over qp list reside on that ibdev, sync with create/destroy qp.*/
2522 	spin_lock_irqsave(&ibdev->reset_flow_resource_lock, flags);
2523 	list_for_each_entry(mqp, &ibdev->qp_list, qps_list) {
2524 		spin_lock_irqsave(&mqp->sq.lock, flags_qp);
2525 		if (mqp->sq.tail != mqp->sq.head) {
2526 			send_mcq = to_mcq(mqp->ibqp.send_cq);
2527 			spin_lock_irqsave(&send_mcq->lock, flags_cq);
2528 			if (send_mcq->mcq.comp &&
2529 			    mqp->ibqp.send_cq->comp_handler) {
2530 				if (!send_mcq->mcq.reset_notify_added) {
2531 					send_mcq->mcq.reset_notify_added = 1;
2532 					list_add_tail(&send_mcq->mcq.reset_notify,
2533 						      &cq_armed_list);
2534 				}
2535 			}
2536 			spin_unlock_irqrestore(&send_mcq->lock, flags_cq);
2537 		}
2538 		spin_unlock_irqrestore(&mqp->sq.lock, flags_qp);
2539 		spin_lock_irqsave(&mqp->rq.lock, flags_qp);
2540 		/* no handling is needed for SRQ */
2541 		if (!mqp->ibqp.srq) {
2542 			if (mqp->rq.tail != mqp->rq.head) {
2543 				recv_mcq = to_mcq(mqp->ibqp.recv_cq);
2544 				spin_lock_irqsave(&recv_mcq->lock, flags_cq);
2545 				if (recv_mcq->mcq.comp &&
2546 				    mqp->ibqp.recv_cq->comp_handler) {
2547 					if (!recv_mcq->mcq.reset_notify_added) {
2548 						recv_mcq->mcq.reset_notify_added = 1;
2549 						list_add_tail(&recv_mcq->mcq.reset_notify,
2550 							      &cq_armed_list);
2551 					}
2552 				}
2553 				spin_unlock_irqrestore(&recv_mcq->lock,
2554 						       flags_cq);
2555 			}
2556 		}
2557 		spin_unlock_irqrestore(&mqp->rq.lock, flags_qp);
2558 	}
2559 	/*At that point all inflight post send were put to be executed as of we
2560 	 * lock/unlock above locks Now need to arm all involved CQs.
2561 	 */
2562 	list_for_each_entry(mcq, &cq_armed_list, reset_notify) {
2563 		mcq->comp(mcq, NULL);
2564 	}
2565 	spin_unlock_irqrestore(&ibdev->reset_flow_resource_lock, flags);
2566 }
2567 
delay_drop_handler(struct work_struct * work)2568 static void delay_drop_handler(struct work_struct *work)
2569 {
2570 	int err;
2571 	struct mlx5_ib_delay_drop *delay_drop =
2572 		container_of(work, struct mlx5_ib_delay_drop,
2573 			     delay_drop_work);
2574 
2575 	atomic_inc(&delay_drop->events_cnt);
2576 
2577 	mutex_lock(&delay_drop->lock);
2578 	err = mlx5_core_set_delay_drop(delay_drop->dev, delay_drop->timeout);
2579 	if (err) {
2580 		mlx5_ib_warn(delay_drop->dev, "Failed to set delay drop, timeout=%u\n",
2581 			     delay_drop->timeout);
2582 		delay_drop->activate = false;
2583 	}
2584 	mutex_unlock(&delay_drop->lock);
2585 }
2586 
handle_general_event(struct mlx5_ib_dev * ibdev,struct mlx5_eqe * eqe,struct ib_event * ibev)2587 static void handle_general_event(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2588 				 struct ib_event *ibev)
2589 {
2590 	u32 port = (eqe->data.port.port >> 4) & 0xf;
2591 
2592 	switch (eqe->sub_type) {
2593 	case MLX5_GENERAL_SUBTYPE_DELAY_DROP_TIMEOUT:
2594 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2595 					    IB_LINK_LAYER_ETHERNET)
2596 			schedule_work(&ibdev->delay_drop.delay_drop_work);
2597 		break;
2598 	default: /* do nothing */
2599 		return;
2600 	}
2601 }
2602 
handle_port_change(struct mlx5_ib_dev * ibdev,struct mlx5_eqe * eqe,struct ib_event * ibev)2603 static int handle_port_change(struct mlx5_ib_dev *ibdev, struct mlx5_eqe *eqe,
2604 			      struct ib_event *ibev)
2605 {
2606 	u32 port = (eqe->data.port.port >> 4) & 0xf;
2607 
2608 	ibev->element.port_num = port;
2609 
2610 	switch (eqe->sub_type) {
2611 	case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE:
2612 	case MLX5_PORT_CHANGE_SUBTYPE_DOWN:
2613 	case MLX5_PORT_CHANGE_SUBTYPE_INITIALIZED:
2614 		/* In RoCE, port up/down events are handled in
2615 		 * mlx5_netdev_event().
2616 		 */
2617 		if (mlx5_ib_port_link_layer(&ibdev->ib_dev, port) ==
2618 					    IB_LINK_LAYER_ETHERNET)
2619 			return -EINVAL;
2620 
2621 		ibev->event = (eqe->sub_type == MLX5_PORT_CHANGE_SUBTYPE_ACTIVE) ?
2622 				IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR;
2623 		break;
2624 
2625 	case MLX5_PORT_CHANGE_SUBTYPE_LID:
2626 		ibev->event = IB_EVENT_LID_CHANGE;
2627 		break;
2628 
2629 	case MLX5_PORT_CHANGE_SUBTYPE_PKEY:
2630 		ibev->event = IB_EVENT_PKEY_CHANGE;
2631 		schedule_work(&ibdev->devr.ports[port - 1].pkey_change_work);
2632 		break;
2633 
2634 	case MLX5_PORT_CHANGE_SUBTYPE_GUID:
2635 		ibev->event = IB_EVENT_GID_CHANGE;
2636 		break;
2637 
2638 	case MLX5_PORT_CHANGE_SUBTYPE_CLIENT_REREG:
2639 		ibev->event = IB_EVENT_CLIENT_REREGISTER;
2640 		break;
2641 	default:
2642 		return -EINVAL;
2643 	}
2644 
2645 	return 0;
2646 }
2647 
mlx5_ib_handle_event(struct work_struct * _work)2648 static void mlx5_ib_handle_event(struct work_struct *_work)
2649 {
2650 	struct mlx5_ib_event_work *work =
2651 		container_of(_work, struct mlx5_ib_event_work, work);
2652 	struct mlx5_ib_dev *ibdev;
2653 	struct ib_event ibev;
2654 	bool fatal = false;
2655 
2656 	if (work->is_slave) {
2657 		ibdev = mlx5_ib_get_ibdev_from_mpi(work->mpi);
2658 		if (!ibdev)
2659 			goto out;
2660 	} else {
2661 		ibdev = work->dev;
2662 	}
2663 
2664 	switch (work->event) {
2665 	case MLX5_DEV_EVENT_SYS_ERROR:
2666 		ibev.event = IB_EVENT_DEVICE_FATAL;
2667 		mlx5_ib_handle_internal_error(ibdev);
2668 		ibev.element.port_num  = (u8)(unsigned long)work->param;
2669 		fatal = true;
2670 		break;
2671 	case MLX5_EVENT_TYPE_PORT_CHANGE:
2672 		if (handle_port_change(ibdev, work->param, &ibev))
2673 			goto out;
2674 		break;
2675 	case MLX5_EVENT_TYPE_GENERAL_EVENT:
2676 		handle_general_event(ibdev, work->param, &ibev);
2677 		fallthrough;
2678 	default:
2679 		goto out;
2680 	}
2681 
2682 	ibev.device = &ibdev->ib_dev;
2683 
2684 	if (!rdma_is_port_valid(&ibdev->ib_dev, ibev.element.port_num)) {
2685 		mlx5_ib_warn(ibdev, "warning: event on port %d\n",  ibev.element.port_num);
2686 		goto out;
2687 	}
2688 
2689 	if (ibdev->ib_active)
2690 		ib_dispatch_event(&ibev);
2691 
2692 	if (fatal)
2693 		ibdev->ib_active = false;
2694 out:
2695 	kfree(work);
2696 }
2697 
mlx5_ib_event(struct notifier_block * nb,unsigned long event,void * param)2698 static int mlx5_ib_event(struct notifier_block *nb,
2699 			 unsigned long event, void *param)
2700 {
2701 	struct mlx5_ib_event_work *work;
2702 
2703 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
2704 	if (!work)
2705 		return NOTIFY_DONE;
2706 
2707 	INIT_WORK(&work->work, mlx5_ib_handle_event);
2708 	work->dev = container_of(nb, struct mlx5_ib_dev, mdev_events);
2709 	work->is_slave = false;
2710 	work->param = param;
2711 	work->event = event;
2712 
2713 	queue_work(mlx5_ib_event_wq, &work->work);
2714 
2715 	return NOTIFY_OK;
2716 }
2717 
mlx5_ib_event_slave_port(struct notifier_block * nb,unsigned long event,void * param)2718 static int mlx5_ib_event_slave_port(struct notifier_block *nb,
2719 				    unsigned long event, void *param)
2720 {
2721 	struct mlx5_ib_event_work *work;
2722 
2723 	work = kmalloc(sizeof(*work), GFP_ATOMIC);
2724 	if (!work)
2725 		return NOTIFY_DONE;
2726 
2727 	INIT_WORK(&work->work, mlx5_ib_handle_event);
2728 	work->mpi = container_of(nb, struct mlx5_ib_multiport_info, mdev_events);
2729 	work->is_slave = true;
2730 	work->param = param;
2731 	work->event = event;
2732 	queue_work(mlx5_ib_event_wq, &work->work);
2733 
2734 	return NOTIFY_OK;
2735 }
2736 
set_has_smi_cap(struct mlx5_ib_dev * dev)2737 static int set_has_smi_cap(struct mlx5_ib_dev *dev)
2738 {
2739 	struct mlx5_hca_vport_context vport_ctx;
2740 	int err;
2741 	int port;
2742 
2743 	if (MLX5_CAP_GEN(dev->mdev, port_type) != MLX5_CAP_PORT_TYPE_IB)
2744 		return 0;
2745 
2746 	for (port = 1; port <= dev->num_ports; port++) {
2747 		if (!MLX5_CAP_GEN(dev->mdev, ib_virt)) {
2748 			dev->port_caps[port - 1].has_smi = true;
2749 			continue;
2750 		}
2751 		err = mlx5_query_hca_vport_context(dev->mdev, 0, port, 0,
2752 						   &vport_ctx);
2753 		if (err) {
2754 			mlx5_ib_err(dev, "query_hca_vport_context for port=%d failed %d\n",
2755 				    port, err);
2756 			return err;
2757 		}
2758 		dev->port_caps[port - 1].has_smi = vport_ctx.has_smi;
2759 	}
2760 
2761 	return 0;
2762 }
2763 
get_ext_port_caps(struct mlx5_ib_dev * dev)2764 static void get_ext_port_caps(struct mlx5_ib_dev *dev)
2765 {
2766 	unsigned int port;
2767 
2768 	rdma_for_each_port (&dev->ib_dev, port)
2769 		mlx5_query_ext_port_caps(dev, port);
2770 }
2771 
mlx5_get_umr_fence(u8 umr_fence_cap)2772 static u8 mlx5_get_umr_fence(u8 umr_fence_cap)
2773 {
2774 	switch (umr_fence_cap) {
2775 	case MLX5_CAP_UMR_FENCE_NONE:
2776 		return MLX5_FENCE_MODE_NONE;
2777 	case MLX5_CAP_UMR_FENCE_SMALL:
2778 		return MLX5_FENCE_MODE_INITIATOR_SMALL;
2779 	default:
2780 		return MLX5_FENCE_MODE_STRONG_ORDERING;
2781 	}
2782 }
2783 
mlx5_ib_dev_res_init(struct mlx5_ib_dev * dev)2784 static int mlx5_ib_dev_res_init(struct mlx5_ib_dev *dev)
2785 {
2786 	struct mlx5_ib_resources *devr = &dev->devr;
2787 	struct ib_srq_init_attr attr;
2788 	struct ib_device *ibdev;
2789 	struct ib_cq_init_attr cq_attr = {.cqe = 1};
2790 	int port;
2791 	int ret = 0;
2792 
2793 	ibdev = &dev->ib_dev;
2794 
2795 	if (!MLX5_CAP_GEN(dev->mdev, xrc))
2796 		return -EOPNOTSUPP;
2797 
2798 	devr->p0 = ib_alloc_pd(ibdev, 0);
2799 	if (IS_ERR(devr->p0))
2800 		return PTR_ERR(devr->p0);
2801 
2802 	devr->c0 = ib_create_cq(ibdev, NULL, NULL, NULL, &cq_attr);
2803 	if (IS_ERR(devr->c0)) {
2804 		ret = PTR_ERR(devr->c0);
2805 		goto error1;
2806 	}
2807 
2808 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn0, 0);
2809 	if (ret)
2810 		goto error2;
2811 
2812 	ret = mlx5_cmd_xrcd_alloc(dev->mdev, &devr->xrcdn1, 0);
2813 	if (ret)
2814 		goto error3;
2815 
2816 	memset(&attr, 0, sizeof(attr));
2817 	attr.attr.max_sge = 1;
2818 	attr.attr.max_wr = 1;
2819 	attr.srq_type = IB_SRQT_XRC;
2820 	attr.ext.cq = devr->c0;
2821 
2822 	devr->s0 = ib_create_srq(devr->p0, &attr);
2823 	if (IS_ERR(devr->s0)) {
2824 		ret = PTR_ERR(devr->s0);
2825 		goto err_create;
2826 	}
2827 
2828 	memset(&attr, 0, sizeof(attr));
2829 	attr.attr.max_sge = 1;
2830 	attr.attr.max_wr = 1;
2831 	attr.srq_type = IB_SRQT_BASIC;
2832 
2833 	devr->s1 = ib_create_srq(devr->p0, &attr);
2834 	if (IS_ERR(devr->s1)) {
2835 		ret = PTR_ERR(devr->s1);
2836 		goto error6;
2837 	}
2838 
2839 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
2840 		INIT_WORK(&devr->ports[port].pkey_change_work,
2841 			  pkey_change_handler);
2842 
2843 	return 0;
2844 
2845 error6:
2846 	ib_destroy_srq(devr->s0);
2847 err_create:
2848 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
2849 error3:
2850 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
2851 error2:
2852 	ib_destroy_cq(devr->c0);
2853 error1:
2854 	ib_dealloc_pd(devr->p0);
2855 	return ret;
2856 }
2857 
mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev * dev)2858 static void mlx5_ib_dev_res_cleanup(struct mlx5_ib_dev *dev)
2859 {
2860 	struct mlx5_ib_resources *devr = &dev->devr;
2861 	int port;
2862 
2863 	/*
2864 	 * Make sure no change P_Key work items are still executing.
2865 	 *
2866 	 * At this stage, the mlx5_ib_event should be unregistered
2867 	 * and it ensures that no new works are added.
2868 	 */
2869 	for (port = 0; port < ARRAY_SIZE(devr->ports); ++port)
2870 		cancel_work_sync(&devr->ports[port].pkey_change_work);
2871 
2872 	ib_destroy_srq(devr->s1);
2873 	ib_destroy_srq(devr->s0);
2874 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn1, 0);
2875 	mlx5_cmd_xrcd_dealloc(dev->mdev, devr->xrcdn0, 0);
2876 	ib_destroy_cq(devr->c0);
2877 	ib_dealloc_pd(devr->p0);
2878 }
2879 
get_core_cap_flags(struct ib_device * ibdev,struct mlx5_hca_vport_context * rep)2880 static u32 get_core_cap_flags(struct ib_device *ibdev,
2881 			      struct mlx5_hca_vport_context *rep)
2882 {
2883 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
2884 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, 1);
2885 	u8 l3_type_cap = MLX5_CAP_ROCE(dev->mdev, l3_type);
2886 	u8 roce_version_cap = MLX5_CAP_ROCE(dev->mdev, roce_version);
2887 	bool raw_support = !mlx5_core_mp_enabled(dev->mdev);
2888 	u32 ret = 0;
2889 
2890 	if (rep->grh_required)
2891 		ret |= RDMA_CORE_CAP_IB_GRH_REQUIRED;
2892 
2893 	if (ll == IB_LINK_LAYER_INFINIBAND)
2894 		return ret | RDMA_CORE_PORT_IBA_IB;
2895 
2896 	if (raw_support)
2897 		ret |= RDMA_CORE_PORT_RAW_PACKET;
2898 
2899 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV4_CAP))
2900 		return ret;
2901 
2902 	if (!(l3_type_cap & MLX5_ROCE_L3_TYPE_IPV6_CAP))
2903 		return ret;
2904 
2905 	if (roce_version_cap & MLX5_ROCE_VERSION_1_CAP)
2906 		ret |= RDMA_CORE_PORT_IBA_ROCE;
2907 
2908 	if (roce_version_cap & MLX5_ROCE_VERSION_2_CAP)
2909 		ret |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
2910 
2911 	return ret;
2912 }
2913 
mlx5_port_immutable(struct ib_device * ibdev,u32 port_num,struct ib_port_immutable * immutable)2914 static int mlx5_port_immutable(struct ib_device *ibdev, u32 port_num,
2915 			       struct ib_port_immutable *immutable)
2916 {
2917 	struct ib_port_attr attr;
2918 	struct mlx5_ib_dev *dev = to_mdev(ibdev);
2919 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(ibdev, port_num);
2920 	struct mlx5_hca_vport_context rep = {0};
2921 	int err;
2922 
2923 	err = ib_query_port(ibdev, port_num, &attr);
2924 	if (err)
2925 		return err;
2926 
2927 	if (ll == IB_LINK_LAYER_INFINIBAND) {
2928 		err = mlx5_query_hca_vport_context(dev->mdev, 0, port_num, 0,
2929 						   &rep);
2930 		if (err)
2931 			return err;
2932 	}
2933 
2934 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
2935 	immutable->gid_tbl_len = attr.gid_tbl_len;
2936 	immutable->core_cap_flags = get_core_cap_flags(ibdev, &rep);
2937 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
2938 
2939 	return 0;
2940 }
2941 
mlx5_port_rep_immutable(struct ib_device * ibdev,u32 port_num,struct ib_port_immutable * immutable)2942 static int mlx5_port_rep_immutable(struct ib_device *ibdev, u32 port_num,
2943 				   struct ib_port_immutable *immutable)
2944 {
2945 	struct ib_port_attr attr;
2946 	int err;
2947 
2948 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
2949 
2950 	err = ib_query_port(ibdev, port_num, &attr);
2951 	if (err)
2952 		return err;
2953 
2954 	immutable->pkey_tbl_len = attr.pkey_tbl_len;
2955 	immutable->gid_tbl_len = attr.gid_tbl_len;
2956 	immutable->core_cap_flags = RDMA_CORE_PORT_RAW_PACKET;
2957 
2958 	return 0;
2959 }
2960 
get_dev_fw_str(struct ib_device * ibdev,char * str)2961 static void get_dev_fw_str(struct ib_device *ibdev, char *str)
2962 {
2963 	struct mlx5_ib_dev *dev =
2964 		container_of(ibdev, struct mlx5_ib_dev, ib_dev);
2965 	snprintf(str, IB_FW_VERSION_NAME_MAX, "%d.%d.%04d",
2966 		 fw_rev_maj(dev->mdev), fw_rev_min(dev->mdev),
2967 		 fw_rev_sub(dev->mdev));
2968 }
2969 
mlx5_eth_lag_init(struct mlx5_ib_dev * dev)2970 static int mlx5_eth_lag_init(struct mlx5_ib_dev *dev)
2971 {
2972 	struct mlx5_core_dev *mdev = dev->mdev;
2973 	struct mlx5_flow_namespace *ns = mlx5_get_flow_namespace(mdev,
2974 								 MLX5_FLOW_NAMESPACE_LAG);
2975 	struct mlx5_flow_table *ft;
2976 	int err;
2977 
2978 	if (!ns || !mlx5_lag_is_active(mdev))
2979 		return 0;
2980 
2981 	err = mlx5_cmd_create_vport_lag(mdev);
2982 	if (err)
2983 		return err;
2984 
2985 	ft = mlx5_create_lag_demux_flow_table(ns, 0, 0);
2986 	if (IS_ERR(ft)) {
2987 		err = PTR_ERR(ft);
2988 		goto err_destroy_vport_lag;
2989 	}
2990 
2991 	dev->flow_db->lag_demux_ft = ft;
2992 	dev->lag_ports = mlx5_lag_get_num_ports(mdev);
2993 	dev->lag_active = true;
2994 	return 0;
2995 
2996 err_destroy_vport_lag:
2997 	mlx5_cmd_destroy_vport_lag(mdev);
2998 	return err;
2999 }
3000 
mlx5_eth_lag_cleanup(struct mlx5_ib_dev * dev)3001 static void mlx5_eth_lag_cleanup(struct mlx5_ib_dev *dev)
3002 {
3003 	struct mlx5_core_dev *mdev = dev->mdev;
3004 
3005 	if (dev->lag_active) {
3006 		dev->lag_active = false;
3007 
3008 		mlx5_destroy_flow_table(dev->flow_db->lag_demux_ft);
3009 		dev->flow_db->lag_demux_ft = NULL;
3010 
3011 		mlx5_cmd_destroy_vport_lag(mdev);
3012 	}
3013 }
3014 
mlx5_add_netdev_notifier(struct mlx5_ib_dev * dev,u32 port_num)3015 static int mlx5_add_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num)
3016 {
3017 	int err;
3018 
3019 	dev->port[port_num].roce.nb.notifier_call = mlx5_netdev_event;
3020 	err = register_netdevice_notifier(&dev->port[port_num].roce.nb);
3021 	if (err) {
3022 		dev->port[port_num].roce.nb.notifier_call = NULL;
3023 		return err;
3024 	}
3025 
3026 	return 0;
3027 }
3028 
mlx5_remove_netdev_notifier(struct mlx5_ib_dev * dev,u32 port_num)3029 static void mlx5_remove_netdev_notifier(struct mlx5_ib_dev *dev, u32 port_num)
3030 {
3031 	if (dev->port[port_num].roce.nb.notifier_call) {
3032 		unregister_netdevice_notifier(&dev->port[port_num].roce.nb);
3033 		dev->port[port_num].roce.nb.notifier_call = NULL;
3034 	}
3035 }
3036 
mlx5_enable_eth(struct mlx5_ib_dev * dev)3037 static int mlx5_enable_eth(struct mlx5_ib_dev *dev)
3038 {
3039 	int err;
3040 
3041 	if (!dev->is_rep && dev->profile != &raw_eth_profile) {
3042 		err = mlx5_nic_vport_enable_roce(dev->mdev);
3043 		if (err)
3044 			return err;
3045 	}
3046 
3047 	err = mlx5_eth_lag_init(dev);
3048 	if (err)
3049 		goto err_disable_roce;
3050 
3051 	return 0;
3052 
3053 err_disable_roce:
3054 	if (!dev->is_rep && dev->profile != &raw_eth_profile)
3055 		mlx5_nic_vport_disable_roce(dev->mdev);
3056 
3057 	return err;
3058 }
3059 
mlx5_disable_eth(struct mlx5_ib_dev * dev)3060 static void mlx5_disable_eth(struct mlx5_ib_dev *dev)
3061 {
3062 	mlx5_eth_lag_cleanup(dev);
3063 	if (!dev->is_rep && dev->profile != &raw_eth_profile)
3064 		mlx5_nic_vport_disable_roce(dev->mdev);
3065 }
3066 
mlx5_ib_rn_get_params(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,struct rdma_netdev_alloc_params * params)3067 static int mlx5_ib_rn_get_params(struct ib_device *device, u32 port_num,
3068 				 enum rdma_netdev_t type,
3069 				 struct rdma_netdev_alloc_params *params)
3070 {
3071 	if (type != RDMA_NETDEV_IPOIB)
3072 		return -EOPNOTSUPP;
3073 
3074 	return mlx5_rdma_rn_get_params(to_mdev(device)->mdev, device, params);
3075 }
3076 
delay_drop_timeout_read(struct file * filp,char __user * buf,size_t count,loff_t * pos)3077 static ssize_t delay_drop_timeout_read(struct file *filp, char __user *buf,
3078 				       size_t count, loff_t *pos)
3079 {
3080 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3081 	char lbuf[20];
3082 	int len;
3083 
3084 	len = snprintf(lbuf, sizeof(lbuf), "%u\n", delay_drop->timeout);
3085 	return simple_read_from_buffer(buf, count, pos, lbuf, len);
3086 }
3087 
delay_drop_timeout_write(struct file * filp,const char __user * buf,size_t count,loff_t * pos)3088 static ssize_t delay_drop_timeout_write(struct file *filp, const char __user *buf,
3089 					size_t count, loff_t *pos)
3090 {
3091 	struct mlx5_ib_delay_drop *delay_drop = filp->private_data;
3092 	u32 timeout;
3093 	u32 var;
3094 
3095 	if (kstrtouint_from_user(buf, count, 0, &var))
3096 		return -EFAULT;
3097 
3098 	timeout = min_t(u32, roundup(var, 100), MLX5_MAX_DELAY_DROP_TIMEOUT_MS *
3099 			1000);
3100 	if (timeout != var)
3101 		mlx5_ib_dbg(delay_drop->dev, "Round delay drop timeout to %u usec\n",
3102 			    timeout);
3103 
3104 	delay_drop->timeout = timeout;
3105 
3106 	return count;
3107 }
3108 
3109 static const struct file_operations fops_delay_drop_timeout = {
3110 	.owner	= THIS_MODULE,
3111 	.open	= simple_open,
3112 	.write	= delay_drop_timeout_write,
3113 	.read	= delay_drop_timeout_read,
3114 };
3115 
mlx5_ib_unbind_slave_port(struct mlx5_ib_dev * ibdev,struct mlx5_ib_multiport_info * mpi)3116 static void mlx5_ib_unbind_slave_port(struct mlx5_ib_dev *ibdev,
3117 				      struct mlx5_ib_multiport_info *mpi)
3118 {
3119 	u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3120 	struct mlx5_ib_port *port = &ibdev->port[port_num];
3121 	int comps;
3122 	int err;
3123 	int i;
3124 
3125 	lockdep_assert_held(&mlx5_ib_multiport_mutex);
3126 
3127 	mlx5_ib_cleanup_cong_debugfs(ibdev, port_num);
3128 
3129 	spin_lock(&port->mp.mpi_lock);
3130 	if (!mpi->ibdev) {
3131 		spin_unlock(&port->mp.mpi_lock);
3132 		return;
3133 	}
3134 
3135 	mpi->ibdev = NULL;
3136 
3137 	spin_unlock(&port->mp.mpi_lock);
3138 	if (mpi->mdev_events.notifier_call)
3139 		mlx5_notifier_unregister(mpi->mdev, &mpi->mdev_events);
3140 	mpi->mdev_events.notifier_call = NULL;
3141 	mlx5_remove_netdev_notifier(ibdev, port_num);
3142 	spin_lock(&port->mp.mpi_lock);
3143 
3144 	comps = mpi->mdev_refcnt;
3145 	if (comps) {
3146 		mpi->unaffiliate = true;
3147 		init_completion(&mpi->unref_comp);
3148 		spin_unlock(&port->mp.mpi_lock);
3149 
3150 		for (i = 0; i < comps; i++)
3151 			wait_for_completion(&mpi->unref_comp);
3152 
3153 		spin_lock(&port->mp.mpi_lock);
3154 		mpi->unaffiliate = false;
3155 	}
3156 
3157 	port->mp.mpi = NULL;
3158 
3159 	spin_unlock(&port->mp.mpi_lock);
3160 
3161 	err = mlx5_nic_vport_unaffiliate_multiport(mpi->mdev);
3162 
3163 	mlx5_ib_dbg(ibdev, "unaffiliated port %u\n", port_num + 1);
3164 	/* Log an error, still needed to cleanup the pointers and add
3165 	 * it back to the list.
3166 	 */
3167 	if (err)
3168 		mlx5_ib_err(ibdev, "Failed to unaffiliate port %u\n",
3169 			    port_num + 1);
3170 
3171 	ibdev->port[port_num].roce.last_port_state = IB_PORT_DOWN;
3172 }
3173 
mlx5_ib_bind_slave_port(struct mlx5_ib_dev * ibdev,struct mlx5_ib_multiport_info * mpi)3174 static bool mlx5_ib_bind_slave_port(struct mlx5_ib_dev *ibdev,
3175 				    struct mlx5_ib_multiport_info *mpi)
3176 {
3177 	u32 port_num = mlx5_core_native_port_num(mpi->mdev) - 1;
3178 	int err;
3179 
3180 	lockdep_assert_held(&mlx5_ib_multiport_mutex);
3181 
3182 	spin_lock(&ibdev->port[port_num].mp.mpi_lock);
3183 	if (ibdev->port[port_num].mp.mpi) {
3184 		mlx5_ib_dbg(ibdev, "port %u already affiliated.\n",
3185 			    port_num + 1);
3186 		spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3187 		return false;
3188 	}
3189 
3190 	ibdev->port[port_num].mp.mpi = mpi;
3191 	mpi->ibdev = ibdev;
3192 	mpi->mdev_events.notifier_call = NULL;
3193 	spin_unlock(&ibdev->port[port_num].mp.mpi_lock);
3194 
3195 	err = mlx5_nic_vport_affiliate_multiport(ibdev->mdev, mpi->mdev);
3196 	if (err)
3197 		goto unbind;
3198 
3199 	err = mlx5_add_netdev_notifier(ibdev, port_num);
3200 	if (err) {
3201 		mlx5_ib_err(ibdev, "failed adding netdev notifier for port %u\n",
3202 			    port_num + 1);
3203 		goto unbind;
3204 	}
3205 
3206 	mpi->mdev_events.notifier_call = mlx5_ib_event_slave_port;
3207 	mlx5_notifier_register(mpi->mdev, &mpi->mdev_events);
3208 
3209 	mlx5_ib_init_cong_debugfs(ibdev, port_num);
3210 
3211 	return true;
3212 
3213 unbind:
3214 	mlx5_ib_unbind_slave_port(ibdev, mpi);
3215 	return false;
3216 }
3217 
mlx5_ib_init_multiport_master(struct mlx5_ib_dev * dev)3218 static int mlx5_ib_init_multiport_master(struct mlx5_ib_dev *dev)
3219 {
3220 	u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3221 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3222 							  port_num + 1);
3223 	struct mlx5_ib_multiport_info *mpi;
3224 	int err;
3225 	u32 i;
3226 
3227 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3228 		return 0;
3229 
3230 	err = mlx5_query_nic_vport_system_image_guid(dev->mdev,
3231 						     &dev->sys_image_guid);
3232 	if (err)
3233 		return err;
3234 
3235 	err = mlx5_nic_vport_enable_roce(dev->mdev);
3236 	if (err)
3237 		return err;
3238 
3239 	mutex_lock(&mlx5_ib_multiport_mutex);
3240 	for (i = 0; i < dev->num_ports; i++) {
3241 		bool bound = false;
3242 
3243 		/* build a stub multiport info struct for the native port. */
3244 		if (i == port_num) {
3245 			mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
3246 			if (!mpi) {
3247 				mutex_unlock(&mlx5_ib_multiport_mutex);
3248 				mlx5_nic_vport_disable_roce(dev->mdev);
3249 				return -ENOMEM;
3250 			}
3251 
3252 			mpi->is_master = true;
3253 			mpi->mdev = dev->mdev;
3254 			mpi->sys_image_guid = dev->sys_image_guid;
3255 			dev->port[i].mp.mpi = mpi;
3256 			mpi->ibdev = dev;
3257 			mpi = NULL;
3258 			continue;
3259 		}
3260 
3261 		list_for_each_entry(mpi, &mlx5_ib_unaffiliated_port_list,
3262 				    list) {
3263 			if (dev->sys_image_guid == mpi->sys_image_guid &&
3264 			    (mlx5_core_native_port_num(mpi->mdev) - 1) == i) {
3265 				bound = mlx5_ib_bind_slave_port(dev, mpi);
3266 			}
3267 
3268 			if (bound) {
3269 				dev_dbg(mpi->mdev->device,
3270 					"removing port from unaffiliated list.\n");
3271 				mlx5_ib_dbg(dev, "port %d bound\n", i + 1);
3272 				list_del(&mpi->list);
3273 				break;
3274 			}
3275 		}
3276 		if (!bound)
3277 			mlx5_ib_dbg(dev, "no free port found for port %d\n",
3278 				    i + 1);
3279 	}
3280 
3281 	list_add_tail(&dev->ib_dev_list, &mlx5_ib_dev_list);
3282 	mutex_unlock(&mlx5_ib_multiport_mutex);
3283 	return err;
3284 }
3285 
mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev * dev)3286 static void mlx5_ib_cleanup_multiport_master(struct mlx5_ib_dev *dev)
3287 {
3288 	u32 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3289 	enum rdma_link_layer ll = mlx5_ib_port_link_layer(&dev->ib_dev,
3290 							  port_num + 1);
3291 	u32 i;
3292 
3293 	if (!mlx5_core_is_mp_master(dev->mdev) || ll != IB_LINK_LAYER_ETHERNET)
3294 		return;
3295 
3296 	mutex_lock(&mlx5_ib_multiport_mutex);
3297 	for (i = 0; i < dev->num_ports; i++) {
3298 		if (dev->port[i].mp.mpi) {
3299 			/* Destroy the native port stub */
3300 			if (i == port_num) {
3301 				kfree(dev->port[i].mp.mpi);
3302 				dev->port[i].mp.mpi = NULL;
3303 			} else {
3304 				mlx5_ib_dbg(dev, "unbinding port_num: %u\n",
3305 					    i + 1);
3306 				list_add_tail(&dev->port[i].mp.mpi->list,
3307 					      &mlx5_ib_unaffiliated_port_list);
3308 				mlx5_ib_unbind_slave_port(dev,
3309 							  dev->port[i].mp.mpi);
3310 			}
3311 		}
3312 	}
3313 
3314 	mlx5_ib_dbg(dev, "removing from devlist\n");
3315 	list_del(&dev->ib_dev_list);
3316 	mutex_unlock(&mlx5_ib_multiport_mutex);
3317 
3318 	mlx5_nic_vport_disable_roce(dev->mdev);
3319 }
3320 
mmap_obj_cleanup(struct ib_uobject * uobject,enum rdma_remove_reason why,struct uverbs_attr_bundle * attrs)3321 static int mmap_obj_cleanup(struct ib_uobject *uobject,
3322 			    enum rdma_remove_reason why,
3323 			    struct uverbs_attr_bundle *attrs)
3324 {
3325 	struct mlx5_user_mmap_entry *obj = uobject->object;
3326 
3327 	rdma_user_mmap_entry_remove(&obj->rdma_entry);
3328 	return 0;
3329 }
3330 
mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext * c,struct mlx5_user_mmap_entry * entry,size_t length)3331 static int mlx5_rdma_user_mmap_entry_insert(struct mlx5_ib_ucontext *c,
3332 					    struct mlx5_user_mmap_entry *entry,
3333 					    size_t length)
3334 {
3335 	return rdma_user_mmap_entry_insert_range(
3336 		&c->ibucontext, &entry->rdma_entry, length,
3337 		(MLX5_IB_MMAP_OFFSET_START << 16),
3338 		((MLX5_IB_MMAP_OFFSET_END << 16) + (1UL << 16) - 1));
3339 }
3340 
3341 static struct mlx5_user_mmap_entry *
alloc_var_entry(struct mlx5_ib_ucontext * c)3342 alloc_var_entry(struct mlx5_ib_ucontext *c)
3343 {
3344 	struct mlx5_user_mmap_entry *entry;
3345 	struct mlx5_var_table *var_table;
3346 	u32 page_idx;
3347 	int err;
3348 
3349 	var_table = &to_mdev(c->ibucontext.device)->var_table;
3350 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3351 	if (!entry)
3352 		return ERR_PTR(-ENOMEM);
3353 
3354 	mutex_lock(&var_table->bitmap_lock);
3355 	page_idx = find_first_zero_bit(var_table->bitmap,
3356 				       var_table->num_var_hw_entries);
3357 	if (page_idx >= var_table->num_var_hw_entries) {
3358 		err = -ENOSPC;
3359 		mutex_unlock(&var_table->bitmap_lock);
3360 		goto end;
3361 	}
3362 
3363 	set_bit(page_idx, var_table->bitmap);
3364 	mutex_unlock(&var_table->bitmap_lock);
3365 
3366 	entry->address = var_table->hw_start_addr +
3367 				(page_idx * var_table->stride_size);
3368 	entry->page_idx = page_idx;
3369 	entry->mmap_flag = MLX5_IB_MMAP_TYPE_VAR;
3370 
3371 	err = mlx5_rdma_user_mmap_entry_insert(c, entry,
3372 					       var_table->stride_size);
3373 	if (err)
3374 		goto err_insert;
3375 
3376 	return entry;
3377 
3378 err_insert:
3379 	mutex_lock(&var_table->bitmap_lock);
3380 	clear_bit(page_idx, var_table->bitmap);
3381 	mutex_unlock(&var_table->bitmap_lock);
3382 end:
3383 	kfree(entry);
3384 	return ERR_PTR(err);
3385 }
3386 
UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)3387 static int UVERBS_HANDLER(MLX5_IB_METHOD_VAR_OBJ_ALLOC)(
3388 	struct uverbs_attr_bundle *attrs)
3389 {
3390 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3391 		attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3392 	struct mlx5_ib_ucontext *c;
3393 	struct mlx5_user_mmap_entry *entry;
3394 	u64 mmap_offset;
3395 	u32 length;
3396 	int err;
3397 
3398 	c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3399 	if (IS_ERR(c))
3400 		return PTR_ERR(c);
3401 
3402 	entry = alloc_var_entry(c);
3403 	if (IS_ERR(entry))
3404 		return PTR_ERR(entry);
3405 
3406 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
3407 	length = entry->rdma_entry.npages * PAGE_SIZE;
3408 	uobj->object = entry;
3409 	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE);
3410 
3411 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3412 			     &mmap_offset, sizeof(mmap_offset));
3413 	if (err)
3414 		return err;
3415 
3416 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3417 			     &entry->page_idx, sizeof(entry->page_idx));
3418 	if (err)
3419 		return err;
3420 
3421 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3422 			     &length, sizeof(length));
3423 	return err;
3424 }
3425 
3426 DECLARE_UVERBS_NAMED_METHOD(
3427 	MLX5_IB_METHOD_VAR_OBJ_ALLOC,
3428 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_ALLOC_HANDLE,
3429 			MLX5_IB_OBJECT_VAR,
3430 			UVERBS_ACCESS_NEW,
3431 			UA_MANDATORY),
3432 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_PAGE_ID,
3433 			   UVERBS_ATTR_TYPE(u32),
3434 			   UA_MANDATORY),
3435 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_LENGTH,
3436 			   UVERBS_ATTR_TYPE(u32),
3437 			   UA_MANDATORY),
3438 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_VAR_OBJ_ALLOC_MMAP_OFFSET,
3439 			    UVERBS_ATTR_TYPE(u64),
3440 			    UA_MANDATORY));
3441 
3442 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3443 	MLX5_IB_METHOD_VAR_OBJ_DESTROY,
3444 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_VAR_OBJ_DESTROY_HANDLE,
3445 			MLX5_IB_OBJECT_VAR,
3446 			UVERBS_ACCESS_DESTROY,
3447 			UA_MANDATORY));
3448 
3449 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_VAR,
3450 			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3451 			    &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_ALLOC),
3452 			    &UVERBS_METHOD(MLX5_IB_METHOD_VAR_OBJ_DESTROY));
3453 
var_is_supported(struct ib_device * device)3454 static bool var_is_supported(struct ib_device *device)
3455 {
3456 	struct mlx5_ib_dev *dev = to_mdev(device);
3457 
3458 	return (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3459 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q);
3460 }
3461 
3462 static struct mlx5_user_mmap_entry *
alloc_uar_entry(struct mlx5_ib_ucontext * c,enum mlx5_ib_uapi_uar_alloc_type alloc_type)3463 alloc_uar_entry(struct mlx5_ib_ucontext *c,
3464 		enum mlx5_ib_uapi_uar_alloc_type alloc_type)
3465 {
3466 	struct mlx5_user_mmap_entry *entry;
3467 	struct mlx5_ib_dev *dev;
3468 	u32 uar_index;
3469 	int err;
3470 
3471 	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
3472 	if (!entry)
3473 		return ERR_PTR(-ENOMEM);
3474 
3475 	dev = to_mdev(c->ibucontext.device);
3476 	err = mlx5_cmd_uar_alloc(dev->mdev, &uar_index, c->devx_uid);
3477 	if (err)
3478 		goto end;
3479 
3480 	entry->page_idx = uar_index;
3481 	entry->address = uar_index2paddress(dev, uar_index);
3482 	if (alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3483 		entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_WC;
3484 	else
3485 		entry->mmap_flag = MLX5_IB_MMAP_TYPE_UAR_NC;
3486 
3487 	err = mlx5_rdma_user_mmap_entry_insert(c, entry, PAGE_SIZE);
3488 	if (err)
3489 		goto err_insert;
3490 
3491 	return entry;
3492 
3493 err_insert:
3494 	mlx5_cmd_uar_dealloc(dev->mdev, uar_index, c->devx_uid);
3495 end:
3496 	kfree(entry);
3497 	return ERR_PTR(err);
3498 }
3499 
UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)3500 static int UVERBS_HANDLER(MLX5_IB_METHOD_UAR_OBJ_ALLOC)(
3501 	struct uverbs_attr_bundle *attrs)
3502 {
3503 	struct ib_uobject *uobj = uverbs_attr_get_uobject(
3504 		attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3505 	enum mlx5_ib_uapi_uar_alloc_type alloc_type;
3506 	struct mlx5_ib_ucontext *c;
3507 	struct mlx5_user_mmap_entry *entry;
3508 	u64 mmap_offset;
3509 	u32 length;
3510 	int err;
3511 
3512 	c = to_mucontext(ib_uverbs_get_ucontext(attrs));
3513 	if (IS_ERR(c))
3514 		return PTR_ERR(c);
3515 
3516 	err = uverbs_get_const(&alloc_type, attrs,
3517 			       MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE);
3518 	if (err)
3519 		return err;
3520 
3521 	if (alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF &&
3522 	    alloc_type != MLX5_IB_UAPI_UAR_ALLOC_TYPE_NC)
3523 		return -EOPNOTSUPP;
3524 
3525 	if (!to_mdev(c->ibucontext.device)->wc_support &&
3526 	    alloc_type == MLX5_IB_UAPI_UAR_ALLOC_TYPE_BF)
3527 		return -EOPNOTSUPP;
3528 
3529 	entry = alloc_uar_entry(c, alloc_type);
3530 	if (IS_ERR(entry))
3531 		return PTR_ERR(entry);
3532 
3533 	mmap_offset = mlx5_entry_to_mmap_offset(entry);
3534 	length = entry->rdma_entry.npages * PAGE_SIZE;
3535 	uobj->object = entry;
3536 	uverbs_finalize_uobj_create(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE);
3537 
3538 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3539 			     &mmap_offset, sizeof(mmap_offset));
3540 	if (err)
3541 		return err;
3542 
3543 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3544 			     &entry->page_idx, sizeof(entry->page_idx));
3545 	if (err)
3546 		return err;
3547 
3548 	err = uverbs_copy_to(attrs, MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3549 			     &length, sizeof(length));
3550 	return err;
3551 }
3552 
3553 DECLARE_UVERBS_NAMED_METHOD(
3554 	MLX5_IB_METHOD_UAR_OBJ_ALLOC,
3555 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_ALLOC_HANDLE,
3556 			MLX5_IB_OBJECT_UAR,
3557 			UVERBS_ACCESS_NEW,
3558 			UA_MANDATORY),
3559 	UVERBS_ATTR_CONST_IN(MLX5_IB_ATTR_UAR_OBJ_ALLOC_TYPE,
3560 			     enum mlx5_ib_uapi_uar_alloc_type,
3561 			     UA_MANDATORY),
3562 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_PAGE_ID,
3563 			   UVERBS_ATTR_TYPE(u32),
3564 			   UA_MANDATORY),
3565 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_LENGTH,
3566 			   UVERBS_ATTR_TYPE(u32),
3567 			   UA_MANDATORY),
3568 	UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_UAR_OBJ_ALLOC_MMAP_OFFSET,
3569 			    UVERBS_ATTR_TYPE(u64),
3570 			    UA_MANDATORY));
3571 
3572 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
3573 	MLX5_IB_METHOD_UAR_OBJ_DESTROY,
3574 	UVERBS_ATTR_IDR(MLX5_IB_ATTR_UAR_OBJ_DESTROY_HANDLE,
3575 			MLX5_IB_OBJECT_UAR,
3576 			UVERBS_ACCESS_DESTROY,
3577 			UA_MANDATORY));
3578 
3579 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_UAR,
3580 			    UVERBS_TYPE_ALLOC_IDR(mmap_obj_cleanup),
3581 			    &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_ALLOC),
3582 			    &UVERBS_METHOD(MLX5_IB_METHOD_UAR_OBJ_DESTROY));
3583 
3584 ADD_UVERBS_ATTRIBUTES_SIMPLE(
3585 	mlx5_ib_query_context,
3586 	UVERBS_OBJECT_DEVICE,
3587 	UVERBS_METHOD_QUERY_CONTEXT,
3588 	UVERBS_ATTR_PTR_OUT(
3589 		MLX5_IB_ATTR_QUERY_CONTEXT_RESP_UCTX,
3590 		UVERBS_ATTR_STRUCT(struct mlx5_ib_alloc_ucontext_resp,
3591 				   dump_fill_mkey),
3592 		UA_MANDATORY));
3593 
3594 static const struct uapi_definition mlx5_ib_defs[] = {
3595 	UAPI_DEF_CHAIN(mlx5_ib_devx_defs),
3596 	UAPI_DEF_CHAIN(mlx5_ib_flow_defs),
3597 	UAPI_DEF_CHAIN(mlx5_ib_qos_defs),
3598 	UAPI_DEF_CHAIN(mlx5_ib_std_types_defs),
3599 	UAPI_DEF_CHAIN(mlx5_ib_dm_defs),
3600 
3601 	UAPI_DEF_CHAIN_OBJ_TREE(UVERBS_OBJECT_DEVICE, &mlx5_ib_query_context),
3602 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_VAR,
3603 				UAPI_DEF_IS_OBJ_SUPPORTED(var_is_supported)),
3604 	UAPI_DEF_CHAIN_OBJ_TREE_NAMED(MLX5_IB_OBJECT_UAR),
3605 	{}
3606 };
3607 
mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev * dev)3608 static void mlx5_ib_stage_init_cleanup(struct mlx5_ib_dev *dev)
3609 {
3610 	mlx5_ib_cleanup_multiport_master(dev);
3611 	WARN_ON(!xa_empty(&dev->odp_mkeys));
3612 	mutex_destroy(&dev->cap_mask_mutex);
3613 	WARN_ON(!xa_empty(&dev->sig_mrs));
3614 	WARN_ON(!bitmap_empty(dev->dm.memic_alloc_pages, MLX5_MAX_MEMIC_PAGES));
3615 }
3616 
mlx5_ib_stage_init_init(struct mlx5_ib_dev * dev)3617 static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev)
3618 {
3619 	struct mlx5_core_dev *mdev = dev->mdev;
3620 	int err;
3621 	int i;
3622 
3623 	dev->ib_dev.node_type = RDMA_NODE_IB_CA;
3624 	dev->ib_dev.local_dma_lkey = 0 /* not supported for now */;
3625 	dev->ib_dev.phys_port_cnt = dev->num_ports;
3626 	dev->ib_dev.dev.parent = mdev->device;
3627 	dev->ib_dev.lag_flags = RDMA_LAG_FLAGS_HASH_ALL_SLAVES;
3628 
3629 	for (i = 0; i < dev->num_ports; i++) {
3630 		spin_lock_init(&dev->port[i].mp.mpi_lock);
3631 		rwlock_init(&dev->port[i].roce.netdev_lock);
3632 		dev->port[i].roce.dev = dev;
3633 		dev->port[i].roce.native_port_num = i + 1;
3634 		dev->port[i].roce.last_port_state = IB_PORT_DOWN;
3635 	}
3636 
3637 	err = mlx5_ib_init_multiport_master(dev);
3638 	if (err)
3639 		return err;
3640 
3641 	err = set_has_smi_cap(dev);
3642 	if (err)
3643 		goto err_mp;
3644 
3645 	err = mlx5_query_max_pkeys(&dev->ib_dev, &dev->pkey_table_len);
3646 	if (err)
3647 		goto err_mp;
3648 
3649 	if (mlx5_use_mad_ifc(dev))
3650 		get_ext_port_caps(dev);
3651 
3652 	dev->ib_dev.num_comp_vectors    = mlx5_comp_vectors_count(mdev);
3653 
3654 	mutex_init(&dev->cap_mask_mutex);
3655 	INIT_LIST_HEAD(&dev->qp_list);
3656 	spin_lock_init(&dev->reset_flow_resource_lock);
3657 	xa_init(&dev->odp_mkeys);
3658 	xa_init(&dev->sig_mrs);
3659 	atomic_set(&dev->mkey_var, 0);
3660 
3661 	spin_lock_init(&dev->dm.lock);
3662 	dev->dm.dev = mdev;
3663 	return 0;
3664 
3665 err_mp:
3666 	mlx5_ib_cleanup_multiport_master(dev);
3667 	return err;
3668 }
3669 
mlx5_ib_enable_driver(struct ib_device * dev)3670 static int mlx5_ib_enable_driver(struct ib_device *dev)
3671 {
3672 	struct mlx5_ib_dev *mdev = to_mdev(dev);
3673 	int ret;
3674 
3675 	ret = mlx5_ib_test_wc(mdev);
3676 	mlx5_ib_dbg(mdev, "Write-Combining %s",
3677 		    mdev->wc_support ? "supported" : "not supported");
3678 
3679 	return ret;
3680 }
3681 
3682 static const struct ib_device_ops mlx5_ib_dev_ops = {
3683 	.owner = THIS_MODULE,
3684 	.driver_id = RDMA_DRIVER_MLX5,
3685 	.uverbs_abi_ver	= MLX5_IB_UVERBS_ABI_VERSION,
3686 
3687 	.add_gid = mlx5_ib_add_gid,
3688 	.alloc_mr = mlx5_ib_alloc_mr,
3689 	.alloc_mr_integrity = mlx5_ib_alloc_mr_integrity,
3690 	.alloc_pd = mlx5_ib_alloc_pd,
3691 	.alloc_ucontext = mlx5_ib_alloc_ucontext,
3692 	.attach_mcast = mlx5_ib_mcg_attach,
3693 	.check_mr_status = mlx5_ib_check_mr_status,
3694 	.create_ah = mlx5_ib_create_ah,
3695 	.create_cq = mlx5_ib_create_cq,
3696 	.create_qp = mlx5_ib_create_qp,
3697 	.create_srq = mlx5_ib_create_srq,
3698 	.create_user_ah = mlx5_ib_create_ah,
3699 	.dealloc_pd = mlx5_ib_dealloc_pd,
3700 	.dealloc_ucontext = mlx5_ib_dealloc_ucontext,
3701 	.del_gid = mlx5_ib_del_gid,
3702 	.dereg_mr = mlx5_ib_dereg_mr,
3703 	.destroy_ah = mlx5_ib_destroy_ah,
3704 	.destroy_cq = mlx5_ib_destroy_cq,
3705 	.destroy_qp = mlx5_ib_destroy_qp,
3706 	.destroy_srq = mlx5_ib_destroy_srq,
3707 	.detach_mcast = mlx5_ib_mcg_detach,
3708 	.disassociate_ucontext = mlx5_ib_disassociate_ucontext,
3709 	.drain_rq = mlx5_ib_drain_rq,
3710 	.drain_sq = mlx5_ib_drain_sq,
3711 	.device_group = &mlx5_attr_group,
3712 	.enable_driver = mlx5_ib_enable_driver,
3713 	.get_dev_fw_str = get_dev_fw_str,
3714 	.get_dma_mr = mlx5_ib_get_dma_mr,
3715 	.get_link_layer = mlx5_ib_port_link_layer,
3716 	.map_mr_sg = mlx5_ib_map_mr_sg,
3717 	.map_mr_sg_pi = mlx5_ib_map_mr_sg_pi,
3718 	.mmap = mlx5_ib_mmap,
3719 	.mmap_free = mlx5_ib_mmap_free,
3720 	.modify_cq = mlx5_ib_modify_cq,
3721 	.modify_device = mlx5_ib_modify_device,
3722 	.modify_port = mlx5_ib_modify_port,
3723 	.modify_qp = mlx5_ib_modify_qp,
3724 	.modify_srq = mlx5_ib_modify_srq,
3725 	.poll_cq = mlx5_ib_poll_cq,
3726 	.post_recv = mlx5_ib_post_recv_nodrain,
3727 	.post_send = mlx5_ib_post_send_nodrain,
3728 	.post_srq_recv = mlx5_ib_post_srq_recv,
3729 	.process_mad = mlx5_ib_process_mad,
3730 	.query_ah = mlx5_ib_query_ah,
3731 	.query_device = mlx5_ib_query_device,
3732 	.query_gid = mlx5_ib_query_gid,
3733 	.query_pkey = mlx5_ib_query_pkey,
3734 	.query_qp = mlx5_ib_query_qp,
3735 	.query_srq = mlx5_ib_query_srq,
3736 	.query_ucontext = mlx5_ib_query_ucontext,
3737 	.reg_user_mr = mlx5_ib_reg_user_mr,
3738 	.reg_user_mr_dmabuf = mlx5_ib_reg_user_mr_dmabuf,
3739 	.req_notify_cq = mlx5_ib_arm_cq,
3740 	.rereg_user_mr = mlx5_ib_rereg_user_mr,
3741 	.resize_cq = mlx5_ib_resize_cq,
3742 
3743 	INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah),
3744 	INIT_RDMA_OBJ_SIZE(ib_counters, mlx5_ib_mcounters, ibcntrs),
3745 	INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq),
3746 	INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd),
3747 	INIT_RDMA_OBJ_SIZE(ib_qp, mlx5_ib_qp, ibqp),
3748 	INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq),
3749 	INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext),
3750 };
3751 
3752 static const struct ib_device_ops mlx5_ib_dev_ipoib_enhanced_ops = {
3753 	.rdma_netdev_get_params = mlx5_ib_rn_get_params,
3754 };
3755 
3756 static const struct ib_device_ops mlx5_ib_dev_sriov_ops = {
3757 	.get_vf_config = mlx5_ib_get_vf_config,
3758 	.get_vf_guid = mlx5_ib_get_vf_guid,
3759 	.get_vf_stats = mlx5_ib_get_vf_stats,
3760 	.set_vf_guid = mlx5_ib_set_vf_guid,
3761 	.set_vf_link_state = mlx5_ib_set_vf_link_state,
3762 };
3763 
3764 static const struct ib_device_ops mlx5_ib_dev_mw_ops = {
3765 	.alloc_mw = mlx5_ib_alloc_mw,
3766 	.dealloc_mw = mlx5_ib_dealloc_mw,
3767 
3768 	INIT_RDMA_OBJ_SIZE(ib_mw, mlx5_ib_mw, ibmw),
3769 };
3770 
3771 static const struct ib_device_ops mlx5_ib_dev_xrc_ops = {
3772 	.alloc_xrcd = mlx5_ib_alloc_xrcd,
3773 	.dealloc_xrcd = mlx5_ib_dealloc_xrcd,
3774 
3775 	INIT_RDMA_OBJ_SIZE(ib_xrcd, mlx5_ib_xrcd, ibxrcd),
3776 };
3777 
mlx5_ib_init_var_table(struct mlx5_ib_dev * dev)3778 static int mlx5_ib_init_var_table(struct mlx5_ib_dev *dev)
3779 {
3780 	struct mlx5_core_dev *mdev = dev->mdev;
3781 	struct mlx5_var_table *var_table = &dev->var_table;
3782 	u8 log_doorbell_bar_size;
3783 	u8 log_doorbell_stride;
3784 	u64 bar_size;
3785 
3786 	log_doorbell_bar_size = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
3787 					log_doorbell_bar_size);
3788 	log_doorbell_stride = MLX5_CAP_DEV_VDPA_EMULATION(mdev,
3789 					log_doorbell_stride);
3790 	var_table->hw_start_addr = dev->mdev->bar_addr +
3791 				MLX5_CAP64_DEV_VDPA_EMULATION(mdev,
3792 					doorbell_bar_offset);
3793 	bar_size = (1ULL << log_doorbell_bar_size) * 4096;
3794 	var_table->stride_size = 1ULL << log_doorbell_stride;
3795 	var_table->num_var_hw_entries = div_u64(bar_size,
3796 						var_table->stride_size);
3797 	mutex_init(&var_table->bitmap_lock);
3798 	var_table->bitmap = bitmap_zalloc(var_table->num_var_hw_entries,
3799 					  GFP_KERNEL);
3800 	return (var_table->bitmap) ? 0 : -ENOMEM;
3801 }
3802 
mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev * dev)3803 static void mlx5_ib_stage_caps_cleanup(struct mlx5_ib_dev *dev)
3804 {
3805 	bitmap_free(dev->var_table.bitmap);
3806 }
3807 
mlx5_ib_stage_caps_init(struct mlx5_ib_dev * dev)3808 static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev)
3809 {
3810 	struct mlx5_core_dev *mdev = dev->mdev;
3811 	int err;
3812 
3813 	if (MLX5_CAP_GEN(mdev, ipoib_enhanced_offloads) &&
3814 	    IS_ENABLED(CONFIG_MLX5_CORE_IPOIB))
3815 		ib_set_device_ops(&dev->ib_dev,
3816 				  &mlx5_ib_dev_ipoib_enhanced_ops);
3817 
3818 	if (mlx5_core_is_pf(mdev))
3819 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_sriov_ops);
3820 
3821 	dev->umr_fence = mlx5_get_umr_fence(MLX5_CAP_GEN(mdev, umr_fence));
3822 
3823 	if (MLX5_CAP_GEN(mdev, imaicl))
3824 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_mw_ops);
3825 
3826 	if (MLX5_CAP_GEN(mdev, xrc))
3827 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_xrc_ops);
3828 
3829 	if (MLX5_CAP_DEV_MEM(mdev, memic) ||
3830 	    MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3831 	    MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)
3832 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_dm_ops);
3833 
3834 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops);
3835 
3836 	if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS))
3837 		dev->ib_dev.driver_def = mlx5_ib_defs;
3838 
3839 	err = init_node_data(dev);
3840 	if (err)
3841 		return err;
3842 
3843 	if ((MLX5_CAP_GEN(dev->mdev, port_type) == MLX5_CAP_PORT_TYPE_ETH) &&
3844 	    (MLX5_CAP_GEN(dev->mdev, disable_local_lb_uc) ||
3845 	     MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc)))
3846 		mutex_init(&dev->lb.mutex);
3847 
3848 	if (MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
3849 			MLX5_GENERAL_OBJ_TYPES_CAP_VIRTIO_NET_Q) {
3850 		err = mlx5_ib_init_var_table(dev);
3851 		if (err)
3852 			return err;
3853 	}
3854 
3855 	dev->ib_dev.use_cq_dim = true;
3856 
3857 	return 0;
3858 }
3859 
3860 static const struct ib_device_ops mlx5_ib_dev_port_ops = {
3861 	.get_port_immutable = mlx5_port_immutable,
3862 	.query_port = mlx5_ib_query_port,
3863 };
3864 
mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev * dev)3865 static int mlx5_ib_stage_non_default_cb(struct mlx5_ib_dev *dev)
3866 {
3867 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_ops);
3868 	return 0;
3869 }
3870 
3871 static const struct ib_device_ops mlx5_ib_dev_port_rep_ops = {
3872 	.get_port_immutable = mlx5_port_rep_immutable,
3873 	.query_port = mlx5_ib_rep_query_port,
3874 	.query_pkey = mlx5_ib_rep_query_pkey,
3875 };
3876 
mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev * dev)3877 static int mlx5_ib_stage_raw_eth_non_default_cb(struct mlx5_ib_dev *dev)
3878 {
3879 	ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_port_rep_ops);
3880 	return 0;
3881 }
3882 
3883 static const struct ib_device_ops mlx5_ib_dev_common_roce_ops = {
3884 	.create_rwq_ind_table = mlx5_ib_create_rwq_ind_table,
3885 	.create_wq = mlx5_ib_create_wq,
3886 	.destroy_rwq_ind_table = mlx5_ib_destroy_rwq_ind_table,
3887 	.destroy_wq = mlx5_ib_destroy_wq,
3888 	.get_netdev = mlx5_ib_get_netdev,
3889 	.modify_wq = mlx5_ib_modify_wq,
3890 
3891 	INIT_RDMA_OBJ_SIZE(ib_rwq_ind_table, mlx5_ib_rwq_ind_table,
3892 			   ib_rwq_ind_tbl),
3893 };
3894 
mlx5_ib_roce_init(struct mlx5_ib_dev * dev)3895 static int mlx5_ib_roce_init(struct mlx5_ib_dev *dev)
3896 {
3897 	struct mlx5_core_dev *mdev = dev->mdev;
3898 	enum rdma_link_layer ll;
3899 	int port_type_cap;
3900 	u32 port_num = 0;
3901 	int err;
3902 
3903 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3904 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
3905 
3906 	if (ll == IB_LINK_LAYER_ETHERNET) {
3907 		ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_common_roce_ops);
3908 
3909 		port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3910 
3911 		/* Register only for native ports */
3912 		err = mlx5_add_netdev_notifier(dev, port_num);
3913 		if (err)
3914 			return err;
3915 
3916 		err = mlx5_enable_eth(dev);
3917 		if (err)
3918 			goto cleanup;
3919 	}
3920 
3921 	return 0;
3922 cleanup:
3923 	mlx5_remove_netdev_notifier(dev, port_num);
3924 	return err;
3925 }
3926 
mlx5_ib_roce_cleanup(struct mlx5_ib_dev * dev)3927 static void mlx5_ib_roce_cleanup(struct mlx5_ib_dev *dev)
3928 {
3929 	struct mlx5_core_dev *mdev = dev->mdev;
3930 	enum rdma_link_layer ll;
3931 	int port_type_cap;
3932 	u32 port_num;
3933 
3934 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
3935 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
3936 
3937 	if (ll == IB_LINK_LAYER_ETHERNET) {
3938 		mlx5_disable_eth(dev);
3939 
3940 		port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3941 		mlx5_remove_netdev_notifier(dev, port_num);
3942 	}
3943 }
3944 
mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev * dev)3945 static int mlx5_ib_stage_cong_debugfs_init(struct mlx5_ib_dev *dev)
3946 {
3947 	mlx5_ib_init_cong_debugfs(dev,
3948 				  mlx5_core_native_port_num(dev->mdev) - 1);
3949 	return 0;
3950 }
3951 
mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev * dev)3952 static void mlx5_ib_stage_cong_debugfs_cleanup(struct mlx5_ib_dev *dev)
3953 {
3954 	mlx5_ib_cleanup_cong_debugfs(dev,
3955 				     mlx5_core_native_port_num(dev->mdev) - 1);
3956 }
3957 
mlx5_ib_stage_uar_init(struct mlx5_ib_dev * dev)3958 static int mlx5_ib_stage_uar_init(struct mlx5_ib_dev *dev)
3959 {
3960 	dev->mdev->priv.uar = mlx5_get_uars_page(dev->mdev);
3961 	return PTR_ERR_OR_ZERO(dev->mdev->priv.uar);
3962 }
3963 
mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev * dev)3964 static void mlx5_ib_stage_uar_cleanup(struct mlx5_ib_dev *dev)
3965 {
3966 	mlx5_put_uars_page(dev->mdev, dev->mdev->priv.uar);
3967 }
3968 
mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev * dev)3969 static int mlx5_ib_stage_bfrag_init(struct mlx5_ib_dev *dev)
3970 {
3971 	int err;
3972 
3973 	err = mlx5_alloc_bfreg(dev->mdev, &dev->bfreg, false, false);
3974 	if (err)
3975 		return err;
3976 
3977 	err = mlx5_alloc_bfreg(dev->mdev, &dev->fp_bfreg, false, true);
3978 	if (err)
3979 		mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3980 
3981 	return err;
3982 }
3983 
mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev * dev)3984 static void mlx5_ib_stage_bfrag_cleanup(struct mlx5_ib_dev *dev)
3985 {
3986 	mlx5_free_bfreg(dev->mdev, &dev->fp_bfreg);
3987 	mlx5_free_bfreg(dev->mdev, &dev->bfreg);
3988 }
3989 
mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev * dev)3990 static int mlx5_ib_stage_ib_reg_init(struct mlx5_ib_dev *dev)
3991 {
3992 	const char *name;
3993 
3994 	if (!mlx5_lag_is_active(dev->mdev))
3995 		name = "mlx5_%d";
3996 	else
3997 		name = "mlx5_bond_%d";
3998 	return ib_register_device(&dev->ib_dev, name, &dev->mdev->pdev->dev);
3999 }
4000 
mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev * dev)4001 static void mlx5_ib_stage_pre_ib_reg_umr_cleanup(struct mlx5_ib_dev *dev)
4002 {
4003 	int err;
4004 
4005 	err = mlx5_mkey_cache_cleanup(dev);
4006 	if (err)
4007 		mlx5_ib_warn(dev, "mr cache cleanup failed\n");
4008 
4009 	mlx5r_umr_resource_cleanup(dev);
4010 }
4011 
mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev * dev)4012 static void mlx5_ib_stage_ib_reg_cleanup(struct mlx5_ib_dev *dev)
4013 {
4014 	ib_unregister_device(&dev->ib_dev);
4015 }
4016 
mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev * dev)4017 static int mlx5_ib_stage_post_ib_reg_umr_init(struct mlx5_ib_dev *dev)
4018 {
4019 	int ret;
4020 
4021 	ret = mlx5r_umr_resource_init(dev);
4022 	if (ret)
4023 		return ret;
4024 
4025 	ret = mlx5_mkey_cache_init(dev);
4026 	if (ret) {
4027 		mlx5_ib_warn(dev, "mr cache init failed %d\n", ret);
4028 		mlx5r_umr_resource_cleanup(dev);
4029 	}
4030 	return ret;
4031 }
4032 
mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev * dev)4033 static int mlx5_ib_stage_delay_drop_init(struct mlx5_ib_dev *dev)
4034 {
4035 	struct dentry *root;
4036 
4037 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4038 		return 0;
4039 
4040 	mutex_init(&dev->delay_drop.lock);
4041 	dev->delay_drop.dev = dev;
4042 	dev->delay_drop.activate = false;
4043 	dev->delay_drop.timeout = MLX5_MAX_DELAY_DROP_TIMEOUT_MS * 1000;
4044 	INIT_WORK(&dev->delay_drop.delay_drop_work, delay_drop_handler);
4045 	atomic_set(&dev->delay_drop.rqs_cnt, 0);
4046 	atomic_set(&dev->delay_drop.events_cnt, 0);
4047 
4048 	if (!mlx5_debugfs_root)
4049 		return 0;
4050 
4051 	root = debugfs_create_dir("delay_drop", mlx5_debugfs_get_dev_root(dev->mdev));
4052 	dev->delay_drop.dir_debugfs = root;
4053 
4054 	debugfs_create_atomic_t("num_timeout_events", 0400, root,
4055 				&dev->delay_drop.events_cnt);
4056 	debugfs_create_atomic_t("num_rqs", 0400, root,
4057 				&dev->delay_drop.rqs_cnt);
4058 	debugfs_create_file("timeout", 0600, root, &dev->delay_drop,
4059 			    &fops_delay_drop_timeout);
4060 	return 0;
4061 }
4062 
mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev * dev)4063 static void mlx5_ib_stage_delay_drop_cleanup(struct mlx5_ib_dev *dev)
4064 {
4065 	if (!(dev->ib_dev.attrs.raw_packet_caps & IB_RAW_PACKET_CAP_DELAY_DROP))
4066 		return;
4067 
4068 	cancel_work_sync(&dev->delay_drop.delay_drop_work);
4069 	if (!dev->delay_drop.dir_debugfs)
4070 		return;
4071 
4072 	debugfs_remove_recursive(dev->delay_drop.dir_debugfs);
4073 	dev->delay_drop.dir_debugfs = NULL;
4074 }
4075 
mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev * dev)4076 static int mlx5_ib_stage_dev_notifier_init(struct mlx5_ib_dev *dev)
4077 {
4078 	dev->mdev_events.notifier_call = mlx5_ib_event;
4079 	mlx5_notifier_register(dev->mdev, &dev->mdev_events);
4080 	return 0;
4081 }
4082 
mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev * dev)4083 static void mlx5_ib_stage_dev_notifier_cleanup(struct mlx5_ib_dev *dev)
4084 {
4085 	mlx5_notifier_unregister(dev->mdev, &dev->mdev_events);
4086 }
4087 
__mlx5_ib_remove(struct mlx5_ib_dev * dev,const struct mlx5_ib_profile * profile,int stage)4088 void __mlx5_ib_remove(struct mlx5_ib_dev *dev,
4089 		      const struct mlx5_ib_profile *profile,
4090 		      int stage)
4091 {
4092 	dev->ib_active = false;
4093 
4094 	/* Number of stages to cleanup */
4095 	while (stage) {
4096 		stage--;
4097 		if (profile->stage[stage].cleanup)
4098 			profile->stage[stage].cleanup(dev);
4099 	}
4100 
4101 	kfree(dev->port);
4102 	ib_dealloc_device(&dev->ib_dev);
4103 }
4104 
__mlx5_ib_add(struct mlx5_ib_dev * dev,const struct mlx5_ib_profile * profile)4105 int __mlx5_ib_add(struct mlx5_ib_dev *dev,
4106 		  const struct mlx5_ib_profile *profile)
4107 {
4108 	int err;
4109 	int i;
4110 
4111 	dev->profile = profile;
4112 
4113 	for (i = 0; i < MLX5_IB_STAGE_MAX; i++) {
4114 		if (profile->stage[i].init) {
4115 			err = profile->stage[i].init(dev);
4116 			if (err)
4117 				goto err_out;
4118 		}
4119 	}
4120 
4121 	dev->ib_active = true;
4122 	return 0;
4123 
4124 err_out:
4125 	/* Clean up stages which were initialized */
4126 	while (i) {
4127 		i--;
4128 		if (profile->stage[i].cleanup)
4129 			profile->stage[i].cleanup(dev);
4130 	}
4131 	return -ENOMEM;
4132 }
4133 
4134 static const struct mlx5_ib_profile pf_profile = {
4135 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
4136 		     mlx5_ib_stage_init_init,
4137 		     mlx5_ib_stage_init_cleanup),
4138 	STAGE_CREATE(MLX5_IB_STAGE_FS,
4139 		     mlx5_ib_fs_init,
4140 		     mlx5_ib_fs_cleanup),
4141 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4142 		     mlx5_ib_stage_caps_init,
4143 		     mlx5_ib_stage_caps_cleanup),
4144 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4145 		     mlx5_ib_stage_non_default_cb,
4146 		     NULL),
4147 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4148 		     mlx5_ib_roce_init,
4149 		     mlx5_ib_roce_cleanup),
4150 	STAGE_CREATE(MLX5_IB_STAGE_QP,
4151 		     mlx5_init_qp_table,
4152 		     mlx5_cleanup_qp_table),
4153 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4154 		     mlx5_init_srq_table,
4155 		     mlx5_cleanup_srq_table),
4156 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4157 		     mlx5_ib_dev_res_init,
4158 		     mlx5_ib_dev_res_cleanup),
4159 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4160 		     mlx5_ib_stage_dev_notifier_init,
4161 		     mlx5_ib_stage_dev_notifier_cleanup),
4162 	STAGE_CREATE(MLX5_IB_STAGE_ODP,
4163 		     mlx5_ib_odp_init_one,
4164 		     mlx5_ib_odp_cleanup_one),
4165 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4166 		     mlx5_ib_counters_init,
4167 		     mlx5_ib_counters_cleanup),
4168 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4169 		     mlx5_ib_stage_cong_debugfs_init,
4170 		     mlx5_ib_stage_cong_debugfs_cleanup),
4171 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
4172 		     mlx5_ib_stage_uar_init,
4173 		     mlx5_ib_stage_uar_cleanup),
4174 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4175 		     mlx5_ib_stage_bfrag_init,
4176 		     mlx5_ib_stage_bfrag_cleanup),
4177 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4178 		     NULL,
4179 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4180 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4181 		     mlx5_ib_devx_init,
4182 		     mlx5_ib_devx_cleanup),
4183 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4184 		     mlx5_ib_stage_ib_reg_init,
4185 		     mlx5_ib_stage_ib_reg_cleanup),
4186 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4187 		     mlx5_ib_stage_post_ib_reg_umr_init,
4188 		     NULL),
4189 	STAGE_CREATE(MLX5_IB_STAGE_DELAY_DROP,
4190 		     mlx5_ib_stage_delay_drop_init,
4191 		     mlx5_ib_stage_delay_drop_cleanup),
4192 	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4193 		     mlx5_ib_restrack_init,
4194 		     NULL),
4195 };
4196 
4197 const struct mlx5_ib_profile raw_eth_profile = {
4198 	STAGE_CREATE(MLX5_IB_STAGE_INIT,
4199 		     mlx5_ib_stage_init_init,
4200 		     mlx5_ib_stage_init_cleanup),
4201 	STAGE_CREATE(MLX5_IB_STAGE_FS,
4202 		     mlx5_ib_fs_init,
4203 		     mlx5_ib_fs_cleanup),
4204 	STAGE_CREATE(MLX5_IB_STAGE_CAPS,
4205 		     mlx5_ib_stage_caps_init,
4206 		     mlx5_ib_stage_caps_cleanup),
4207 	STAGE_CREATE(MLX5_IB_STAGE_NON_DEFAULT_CB,
4208 		     mlx5_ib_stage_raw_eth_non_default_cb,
4209 		     NULL),
4210 	STAGE_CREATE(MLX5_IB_STAGE_ROCE,
4211 		     mlx5_ib_roce_init,
4212 		     mlx5_ib_roce_cleanup),
4213 	STAGE_CREATE(MLX5_IB_STAGE_QP,
4214 		     mlx5_init_qp_table,
4215 		     mlx5_cleanup_qp_table),
4216 	STAGE_CREATE(MLX5_IB_STAGE_SRQ,
4217 		     mlx5_init_srq_table,
4218 		     mlx5_cleanup_srq_table),
4219 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_RESOURCES,
4220 		     mlx5_ib_dev_res_init,
4221 		     mlx5_ib_dev_res_cleanup),
4222 	STAGE_CREATE(MLX5_IB_STAGE_DEVICE_NOTIFIER,
4223 		     mlx5_ib_stage_dev_notifier_init,
4224 		     mlx5_ib_stage_dev_notifier_cleanup),
4225 	STAGE_CREATE(MLX5_IB_STAGE_COUNTERS,
4226 		     mlx5_ib_counters_init,
4227 		     mlx5_ib_counters_cleanup),
4228 	STAGE_CREATE(MLX5_IB_STAGE_CONG_DEBUGFS,
4229 		     mlx5_ib_stage_cong_debugfs_init,
4230 		     mlx5_ib_stage_cong_debugfs_cleanup),
4231 	STAGE_CREATE(MLX5_IB_STAGE_UAR,
4232 		     mlx5_ib_stage_uar_init,
4233 		     mlx5_ib_stage_uar_cleanup),
4234 	STAGE_CREATE(MLX5_IB_STAGE_BFREG,
4235 		     mlx5_ib_stage_bfrag_init,
4236 		     mlx5_ib_stage_bfrag_cleanup),
4237 	STAGE_CREATE(MLX5_IB_STAGE_PRE_IB_REG_UMR,
4238 		     NULL,
4239 		     mlx5_ib_stage_pre_ib_reg_umr_cleanup),
4240 	STAGE_CREATE(MLX5_IB_STAGE_WHITELIST_UID,
4241 		     mlx5_ib_devx_init,
4242 		     mlx5_ib_devx_cleanup),
4243 	STAGE_CREATE(MLX5_IB_STAGE_IB_REG,
4244 		     mlx5_ib_stage_ib_reg_init,
4245 		     mlx5_ib_stage_ib_reg_cleanup),
4246 	STAGE_CREATE(MLX5_IB_STAGE_POST_IB_REG_UMR,
4247 		     mlx5_ib_stage_post_ib_reg_umr_init,
4248 		     NULL),
4249 	STAGE_CREATE(MLX5_IB_STAGE_RESTRACK,
4250 		     mlx5_ib_restrack_init,
4251 		     NULL),
4252 };
4253 
mlx5r_mp_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)4254 static int mlx5r_mp_probe(struct auxiliary_device *adev,
4255 			  const struct auxiliary_device_id *id)
4256 {
4257 	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4258 	struct mlx5_core_dev *mdev = idev->mdev;
4259 	struct mlx5_ib_multiport_info *mpi;
4260 	struct mlx5_ib_dev *dev;
4261 	bool bound = false;
4262 	int err;
4263 
4264 	mpi = kzalloc(sizeof(*mpi), GFP_KERNEL);
4265 	if (!mpi)
4266 		return -ENOMEM;
4267 
4268 	mpi->mdev = mdev;
4269 	err = mlx5_query_nic_vport_system_image_guid(mdev,
4270 						     &mpi->sys_image_guid);
4271 	if (err) {
4272 		kfree(mpi);
4273 		return err;
4274 	}
4275 
4276 	mutex_lock(&mlx5_ib_multiport_mutex);
4277 	list_for_each_entry(dev, &mlx5_ib_dev_list, ib_dev_list) {
4278 		if (dev->sys_image_guid == mpi->sys_image_guid)
4279 			bound = mlx5_ib_bind_slave_port(dev, mpi);
4280 
4281 		if (bound) {
4282 			rdma_roce_rescan_device(&dev->ib_dev);
4283 			mpi->ibdev->ib_active = true;
4284 			break;
4285 		}
4286 	}
4287 
4288 	if (!bound) {
4289 		list_add_tail(&mpi->list, &mlx5_ib_unaffiliated_port_list);
4290 		dev_dbg(mdev->device,
4291 			"no suitable IB device found to bind to, added to unaffiliated list.\n");
4292 	}
4293 	mutex_unlock(&mlx5_ib_multiport_mutex);
4294 
4295 	auxiliary_set_drvdata(adev, mpi);
4296 	return 0;
4297 }
4298 
mlx5r_mp_remove(struct auxiliary_device * adev)4299 static void mlx5r_mp_remove(struct auxiliary_device *adev)
4300 {
4301 	struct mlx5_ib_multiport_info *mpi;
4302 
4303 	mpi = auxiliary_get_drvdata(adev);
4304 	mutex_lock(&mlx5_ib_multiport_mutex);
4305 	if (mpi->ibdev)
4306 		mlx5_ib_unbind_slave_port(mpi->ibdev, mpi);
4307 	else
4308 		list_del(&mpi->list);
4309 	mutex_unlock(&mlx5_ib_multiport_mutex);
4310 	kfree(mpi);
4311 }
4312 
mlx5r_probe(struct auxiliary_device * adev,const struct auxiliary_device_id * id)4313 static int mlx5r_probe(struct auxiliary_device *adev,
4314 		       const struct auxiliary_device_id *id)
4315 {
4316 	struct mlx5_adev *idev = container_of(adev, struct mlx5_adev, adev);
4317 	struct mlx5_core_dev *mdev = idev->mdev;
4318 	const struct mlx5_ib_profile *profile;
4319 	int port_type_cap, num_ports, ret;
4320 	enum rdma_link_layer ll;
4321 	struct mlx5_ib_dev *dev;
4322 
4323 	port_type_cap = MLX5_CAP_GEN(mdev, port_type);
4324 	ll = mlx5_port_type_cap_to_rdma_ll(port_type_cap);
4325 
4326 	num_ports = max(MLX5_CAP_GEN(mdev, num_ports),
4327 			MLX5_CAP_GEN(mdev, num_vhca_ports));
4328 	dev = ib_alloc_device(mlx5_ib_dev, ib_dev);
4329 	if (!dev)
4330 		return -ENOMEM;
4331 	dev->port = kcalloc(num_ports, sizeof(*dev->port),
4332 			     GFP_KERNEL);
4333 	if (!dev->port) {
4334 		ib_dealloc_device(&dev->ib_dev);
4335 		return -ENOMEM;
4336 	}
4337 
4338 	dev->mdev = mdev;
4339 	dev->num_ports = num_ports;
4340 
4341 	if (ll == IB_LINK_LAYER_ETHERNET && !mlx5_get_roce_state(mdev))
4342 		profile = &raw_eth_profile;
4343 	else
4344 		profile = &pf_profile;
4345 
4346 	ret = __mlx5_ib_add(dev, profile);
4347 	if (ret) {
4348 		kfree(dev->port);
4349 		ib_dealloc_device(&dev->ib_dev);
4350 		return ret;
4351 	}
4352 
4353 	auxiliary_set_drvdata(adev, dev);
4354 	return 0;
4355 }
4356 
mlx5r_remove(struct auxiliary_device * adev)4357 static void mlx5r_remove(struct auxiliary_device *adev)
4358 {
4359 	struct mlx5_ib_dev *dev;
4360 
4361 	dev = auxiliary_get_drvdata(adev);
4362 	__mlx5_ib_remove(dev, dev->profile, MLX5_IB_STAGE_MAX);
4363 }
4364 
4365 static const struct auxiliary_device_id mlx5r_mp_id_table[] = {
4366 	{ .name = MLX5_ADEV_NAME ".multiport", },
4367 	{},
4368 };
4369 
4370 static const struct auxiliary_device_id mlx5r_id_table[] = {
4371 	{ .name = MLX5_ADEV_NAME ".rdma", },
4372 	{},
4373 };
4374 
4375 MODULE_DEVICE_TABLE(auxiliary, mlx5r_mp_id_table);
4376 MODULE_DEVICE_TABLE(auxiliary, mlx5r_id_table);
4377 
4378 static struct auxiliary_driver mlx5r_mp_driver = {
4379 	.name = "multiport",
4380 	.probe = mlx5r_mp_probe,
4381 	.remove = mlx5r_mp_remove,
4382 	.id_table = mlx5r_mp_id_table,
4383 };
4384 
4385 static struct auxiliary_driver mlx5r_driver = {
4386 	.name = "rdma",
4387 	.probe = mlx5r_probe,
4388 	.remove = mlx5r_remove,
4389 	.id_table = mlx5r_id_table,
4390 };
4391 
mlx5_ib_init(void)4392 static int __init mlx5_ib_init(void)
4393 {
4394 	int ret;
4395 
4396 	xlt_emergency_page = (void *)__get_free_page(GFP_KERNEL);
4397 	if (!xlt_emergency_page)
4398 		return -ENOMEM;
4399 
4400 	mlx5_ib_event_wq = alloc_ordered_workqueue("mlx5_ib_event_wq", 0);
4401 	if (!mlx5_ib_event_wq) {
4402 		free_page((unsigned long)xlt_emergency_page);
4403 		return -ENOMEM;
4404 	}
4405 
4406 	mlx5_ib_odp_init();
4407 	ret = mlx5r_rep_init();
4408 	if (ret)
4409 		goto rep_err;
4410 	ret = auxiliary_driver_register(&mlx5r_mp_driver);
4411 	if (ret)
4412 		goto mp_err;
4413 	ret = auxiliary_driver_register(&mlx5r_driver);
4414 	if (ret)
4415 		goto drv_err;
4416 	return 0;
4417 
4418 drv_err:
4419 	auxiliary_driver_unregister(&mlx5r_mp_driver);
4420 mp_err:
4421 	mlx5r_rep_cleanup();
4422 rep_err:
4423 	destroy_workqueue(mlx5_ib_event_wq);
4424 	free_page((unsigned long)xlt_emergency_page);
4425 	return ret;
4426 }
4427 
mlx5_ib_cleanup(void)4428 static void __exit mlx5_ib_cleanup(void)
4429 {
4430 	auxiliary_driver_unregister(&mlx5r_driver);
4431 	auxiliary_driver_unregister(&mlx5r_mp_driver);
4432 	mlx5r_rep_cleanup();
4433 
4434 	destroy_workqueue(mlx5_ib_event_wq);
4435 	free_page((unsigned long)xlt_emergency_page);
4436 }
4437 
4438 module_init(mlx5_ib_init);
4439 module_exit(mlx5_ib_cleanup);
4440