1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  */
38 
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
44 #include <linux/in.h>
45 #include <linux/in6.h>
46 #include <net/addrconf.h>
47 #include <linux/security.h>
48 
49 #include <rdma/ib_verbs.h>
50 #include <rdma/ib_cache.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/rw.h>
53 #include <rdma/lag.h>
54 
55 #include "core_priv.h"
56 #include <trace/events/rdma_core.h>
57 
58 static int ib_resolve_eth_dmac(struct ib_device *device,
59 			       struct rdma_ah_attr *ah_attr);
60 
61 static const char * const ib_events[] = {
62 	[IB_EVENT_CQ_ERR]		= "CQ error",
63 	[IB_EVENT_QP_FATAL]		= "QP fatal error",
64 	[IB_EVENT_QP_REQ_ERR]		= "QP request error",
65 	[IB_EVENT_QP_ACCESS_ERR]	= "QP access error",
66 	[IB_EVENT_COMM_EST]		= "communication established",
67 	[IB_EVENT_SQ_DRAINED]		= "send queue drained",
68 	[IB_EVENT_PATH_MIG]		= "path migration successful",
69 	[IB_EVENT_PATH_MIG_ERR]		= "path migration error",
70 	[IB_EVENT_DEVICE_FATAL]		= "device fatal error",
71 	[IB_EVENT_PORT_ACTIVE]		= "port active",
72 	[IB_EVENT_PORT_ERR]		= "port error",
73 	[IB_EVENT_LID_CHANGE]		= "LID change",
74 	[IB_EVENT_PKEY_CHANGE]		= "P_key change",
75 	[IB_EVENT_SM_CHANGE]		= "SM change",
76 	[IB_EVENT_SRQ_ERR]		= "SRQ error",
77 	[IB_EVENT_SRQ_LIMIT_REACHED]	= "SRQ limit reached",
78 	[IB_EVENT_QP_LAST_WQE_REACHED]	= "last WQE reached",
79 	[IB_EVENT_CLIENT_REREGISTER]	= "client reregister",
80 	[IB_EVENT_GID_CHANGE]		= "GID changed",
81 };
82 
ib_event_msg(enum ib_event_type event)83 const char *__attribute_const__ ib_event_msg(enum ib_event_type event)
84 {
85 	size_t index = event;
86 
87 	return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ?
88 			ib_events[index] : "unrecognized event";
89 }
90 EXPORT_SYMBOL(ib_event_msg);
91 
92 static const char * const wc_statuses[] = {
93 	[IB_WC_SUCCESS]			= "success",
94 	[IB_WC_LOC_LEN_ERR]		= "local length error",
95 	[IB_WC_LOC_QP_OP_ERR]		= "local QP operation error",
96 	[IB_WC_LOC_EEC_OP_ERR]		= "local EE context operation error",
97 	[IB_WC_LOC_PROT_ERR]		= "local protection error",
98 	[IB_WC_WR_FLUSH_ERR]		= "WR flushed",
99 	[IB_WC_MW_BIND_ERR]		= "memory bind operation error",
100 	[IB_WC_BAD_RESP_ERR]		= "bad response error",
101 	[IB_WC_LOC_ACCESS_ERR]		= "local access error",
102 	[IB_WC_REM_INV_REQ_ERR]		= "remote invalid request error",
103 	[IB_WC_REM_ACCESS_ERR]		= "remote access error",
104 	[IB_WC_REM_OP_ERR]		= "remote operation error",
105 	[IB_WC_RETRY_EXC_ERR]		= "transport retry counter exceeded",
106 	[IB_WC_RNR_RETRY_EXC_ERR]	= "RNR retry counter exceeded",
107 	[IB_WC_LOC_RDD_VIOL_ERR]	= "local RDD violation error",
108 	[IB_WC_REM_INV_RD_REQ_ERR]	= "remote invalid RD request",
109 	[IB_WC_REM_ABORT_ERR]		= "operation aborted",
110 	[IB_WC_INV_EECN_ERR]		= "invalid EE context number",
111 	[IB_WC_INV_EEC_STATE_ERR]	= "invalid EE context state",
112 	[IB_WC_FATAL_ERR]		= "fatal error",
113 	[IB_WC_RESP_TIMEOUT_ERR]	= "response timeout error",
114 	[IB_WC_GENERAL_ERR]		= "general error",
115 };
116 
ib_wc_status_msg(enum ib_wc_status status)117 const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status)
118 {
119 	size_t index = status;
120 
121 	return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ?
122 			wc_statuses[index] : "unrecognized status";
123 }
124 EXPORT_SYMBOL(ib_wc_status_msg);
125 
ib_rate_to_mult(enum ib_rate rate)126 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
127 {
128 	switch (rate) {
129 	case IB_RATE_2_5_GBPS: return   1;
130 	case IB_RATE_5_GBPS:   return   2;
131 	case IB_RATE_10_GBPS:  return   4;
132 	case IB_RATE_20_GBPS:  return   8;
133 	case IB_RATE_30_GBPS:  return  12;
134 	case IB_RATE_40_GBPS:  return  16;
135 	case IB_RATE_60_GBPS:  return  24;
136 	case IB_RATE_80_GBPS:  return  32;
137 	case IB_RATE_120_GBPS: return  48;
138 	case IB_RATE_14_GBPS:  return   6;
139 	case IB_RATE_56_GBPS:  return  22;
140 	case IB_RATE_112_GBPS: return  45;
141 	case IB_RATE_168_GBPS: return  67;
142 	case IB_RATE_25_GBPS:  return  10;
143 	case IB_RATE_100_GBPS: return  40;
144 	case IB_RATE_200_GBPS: return  80;
145 	case IB_RATE_300_GBPS: return 120;
146 	case IB_RATE_28_GBPS:  return  11;
147 	case IB_RATE_50_GBPS:  return  20;
148 	case IB_RATE_400_GBPS: return 160;
149 	case IB_RATE_600_GBPS: return 240;
150 	default:	       return  -1;
151 	}
152 }
153 EXPORT_SYMBOL(ib_rate_to_mult);
154 
mult_to_ib_rate(int mult)155 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
156 {
157 	switch (mult) {
158 	case 1:   return IB_RATE_2_5_GBPS;
159 	case 2:   return IB_RATE_5_GBPS;
160 	case 4:   return IB_RATE_10_GBPS;
161 	case 8:   return IB_RATE_20_GBPS;
162 	case 12:  return IB_RATE_30_GBPS;
163 	case 16:  return IB_RATE_40_GBPS;
164 	case 24:  return IB_RATE_60_GBPS;
165 	case 32:  return IB_RATE_80_GBPS;
166 	case 48:  return IB_RATE_120_GBPS;
167 	case 6:   return IB_RATE_14_GBPS;
168 	case 22:  return IB_RATE_56_GBPS;
169 	case 45:  return IB_RATE_112_GBPS;
170 	case 67:  return IB_RATE_168_GBPS;
171 	case 10:  return IB_RATE_25_GBPS;
172 	case 40:  return IB_RATE_100_GBPS;
173 	case 80:  return IB_RATE_200_GBPS;
174 	case 120: return IB_RATE_300_GBPS;
175 	case 11:  return IB_RATE_28_GBPS;
176 	case 20:  return IB_RATE_50_GBPS;
177 	case 160: return IB_RATE_400_GBPS;
178 	case 240: return IB_RATE_600_GBPS;
179 	default:  return IB_RATE_PORT_CURRENT;
180 	}
181 }
182 EXPORT_SYMBOL(mult_to_ib_rate);
183 
ib_rate_to_mbps(enum ib_rate rate)184 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
185 {
186 	switch (rate) {
187 	case IB_RATE_2_5_GBPS: return 2500;
188 	case IB_RATE_5_GBPS:   return 5000;
189 	case IB_RATE_10_GBPS:  return 10000;
190 	case IB_RATE_20_GBPS:  return 20000;
191 	case IB_RATE_30_GBPS:  return 30000;
192 	case IB_RATE_40_GBPS:  return 40000;
193 	case IB_RATE_60_GBPS:  return 60000;
194 	case IB_RATE_80_GBPS:  return 80000;
195 	case IB_RATE_120_GBPS: return 120000;
196 	case IB_RATE_14_GBPS:  return 14062;
197 	case IB_RATE_56_GBPS:  return 56250;
198 	case IB_RATE_112_GBPS: return 112500;
199 	case IB_RATE_168_GBPS: return 168750;
200 	case IB_RATE_25_GBPS:  return 25781;
201 	case IB_RATE_100_GBPS: return 103125;
202 	case IB_RATE_200_GBPS: return 206250;
203 	case IB_RATE_300_GBPS: return 309375;
204 	case IB_RATE_28_GBPS:  return 28125;
205 	case IB_RATE_50_GBPS:  return 53125;
206 	case IB_RATE_400_GBPS: return 425000;
207 	case IB_RATE_600_GBPS: return 637500;
208 	default:	       return -1;
209 	}
210 }
211 EXPORT_SYMBOL(ib_rate_to_mbps);
212 
213 __attribute_const__ enum rdma_transport_type
rdma_node_get_transport(unsigned int node_type)214 rdma_node_get_transport(unsigned int node_type)
215 {
216 
217 	if (node_type == RDMA_NODE_USNIC)
218 		return RDMA_TRANSPORT_USNIC;
219 	if (node_type == RDMA_NODE_USNIC_UDP)
220 		return RDMA_TRANSPORT_USNIC_UDP;
221 	if (node_type == RDMA_NODE_RNIC)
222 		return RDMA_TRANSPORT_IWARP;
223 	if (node_type == RDMA_NODE_UNSPECIFIED)
224 		return RDMA_TRANSPORT_UNSPECIFIED;
225 
226 	return RDMA_TRANSPORT_IB;
227 }
228 EXPORT_SYMBOL(rdma_node_get_transport);
229 
rdma_port_get_link_layer(struct ib_device * device,u32 port_num)230 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
231 					      u32 port_num)
232 {
233 	enum rdma_transport_type lt;
234 	if (device->ops.get_link_layer)
235 		return device->ops.get_link_layer(device, port_num);
236 
237 	lt = rdma_node_get_transport(device->node_type);
238 	if (lt == RDMA_TRANSPORT_IB)
239 		return IB_LINK_LAYER_INFINIBAND;
240 
241 	return IB_LINK_LAYER_ETHERNET;
242 }
243 EXPORT_SYMBOL(rdma_port_get_link_layer);
244 
245 /* Protection domains */
246 
247 /**
248  * __ib_alloc_pd - Allocates an unused protection domain.
249  * @device: The device on which to allocate the protection domain.
250  * @flags: protection domain flags
251  * @caller: caller's build-time module name
252  *
253  * A protection domain object provides an association between QPs, shared
254  * receive queues, address handles, memory regions, and memory windows.
255  *
256  * Every PD has a local_dma_lkey which can be used as the lkey value for local
257  * memory operations.
258  */
__ib_alloc_pd(struct ib_device * device,unsigned int flags,const char * caller)259 struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
260 		const char *caller)
261 {
262 	struct ib_pd *pd;
263 	int mr_access_flags = 0;
264 	int ret;
265 
266 	pd = rdma_zalloc_drv_obj(device, ib_pd);
267 	if (!pd)
268 		return ERR_PTR(-ENOMEM);
269 
270 	pd->device = device;
271 	pd->flags = flags;
272 
273 	rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
274 	rdma_restrack_set_name(&pd->res, caller);
275 
276 	ret = device->ops.alloc_pd(pd, NULL);
277 	if (ret) {
278 		rdma_restrack_put(&pd->res);
279 		kfree(pd);
280 		return ERR_PTR(ret);
281 	}
282 	rdma_restrack_add(&pd->res);
283 
284 	if (device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY)
285 		pd->local_dma_lkey = device->local_dma_lkey;
286 	else
287 		mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
288 
289 	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
290 		pr_warn("%s: enabling unsafe global rkey\n", caller);
291 		mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
292 	}
293 
294 	if (mr_access_flags) {
295 		struct ib_mr *mr;
296 
297 		mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
298 		if (IS_ERR(mr)) {
299 			ib_dealloc_pd(pd);
300 			return ERR_CAST(mr);
301 		}
302 
303 		mr->device	= pd->device;
304 		mr->pd		= pd;
305 		mr->type        = IB_MR_TYPE_DMA;
306 		mr->uobject	= NULL;
307 		mr->need_inval	= false;
308 
309 		pd->__internal_mr = mr;
310 
311 		if (!(device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY))
312 			pd->local_dma_lkey = pd->__internal_mr->lkey;
313 
314 		if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
315 			pd->unsafe_global_rkey = pd->__internal_mr->rkey;
316 	}
317 
318 	return pd;
319 }
320 EXPORT_SYMBOL(__ib_alloc_pd);
321 
322 /**
323  * ib_dealloc_pd_user - Deallocates a protection domain.
324  * @pd: The protection domain to deallocate.
325  * @udata: Valid user data or NULL for kernel object
326  *
327  * It is an error to call this function while any resources in the pd still
328  * exist.  The caller is responsible to synchronously destroy them and
329  * guarantee no new allocations will happen.
330  */
ib_dealloc_pd_user(struct ib_pd * pd,struct ib_udata * udata)331 int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
332 {
333 	int ret;
334 
335 	if (pd->__internal_mr) {
336 		ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
337 		WARN_ON(ret);
338 		pd->__internal_mr = NULL;
339 	}
340 
341 	ret = pd->device->ops.dealloc_pd(pd, udata);
342 	if (ret)
343 		return ret;
344 
345 	rdma_restrack_del(&pd->res);
346 	kfree(pd);
347 	return ret;
348 }
349 EXPORT_SYMBOL(ib_dealloc_pd_user);
350 
351 /* Address handles */
352 
353 /**
354  * rdma_copy_ah_attr - Copy rdma ah attribute from source to destination.
355  * @dest:       Pointer to destination ah_attr. Contents of the destination
356  *              pointer is assumed to be invalid and attribute are overwritten.
357  * @src:        Pointer to source ah_attr.
358  */
rdma_copy_ah_attr(struct rdma_ah_attr * dest,const struct rdma_ah_attr * src)359 void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
360 		       const struct rdma_ah_attr *src)
361 {
362 	*dest = *src;
363 	if (dest->grh.sgid_attr)
364 		rdma_hold_gid_attr(dest->grh.sgid_attr);
365 }
366 EXPORT_SYMBOL(rdma_copy_ah_attr);
367 
368 /**
369  * rdma_replace_ah_attr - Replace valid ah_attr with new new one.
370  * @old:        Pointer to existing ah_attr which needs to be replaced.
371  *              old is assumed to be valid or zero'd
372  * @new:        Pointer to the new ah_attr.
373  *
374  * rdma_replace_ah_attr() first releases any reference in the old ah_attr if
375  * old the ah_attr is valid; after that it copies the new attribute and holds
376  * the reference to the replaced ah_attr.
377  */
rdma_replace_ah_attr(struct rdma_ah_attr * old,const struct rdma_ah_attr * new)378 void rdma_replace_ah_attr(struct rdma_ah_attr *old,
379 			  const struct rdma_ah_attr *new)
380 {
381 	rdma_destroy_ah_attr(old);
382 	*old = *new;
383 	if (old->grh.sgid_attr)
384 		rdma_hold_gid_attr(old->grh.sgid_attr);
385 }
386 EXPORT_SYMBOL(rdma_replace_ah_attr);
387 
388 /**
389  * rdma_move_ah_attr - Move ah_attr pointed by source to destination.
390  * @dest:       Pointer to destination ah_attr to copy to.
391  *              dest is assumed to be valid or zero'd
392  * @src:        Pointer to the new ah_attr.
393  *
394  * rdma_move_ah_attr() first releases any reference in the destination ah_attr
395  * if it is valid. This also transfers ownership of internal references from
396  * src to dest, making src invalid in the process. No new reference of the src
397  * ah_attr is taken.
398  */
rdma_move_ah_attr(struct rdma_ah_attr * dest,struct rdma_ah_attr * src)399 void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src)
400 {
401 	rdma_destroy_ah_attr(dest);
402 	*dest = *src;
403 	src->grh.sgid_attr = NULL;
404 }
405 EXPORT_SYMBOL(rdma_move_ah_attr);
406 
407 /*
408  * Validate that the rdma_ah_attr is valid for the device before passing it
409  * off to the driver.
410  */
rdma_check_ah_attr(struct ib_device * device,struct rdma_ah_attr * ah_attr)411 static int rdma_check_ah_attr(struct ib_device *device,
412 			      struct rdma_ah_attr *ah_attr)
413 {
414 	if (!rdma_is_port_valid(device, ah_attr->port_num))
415 		return -EINVAL;
416 
417 	if ((rdma_is_grh_required(device, ah_attr->port_num) ||
418 	     ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) &&
419 	    !(ah_attr->ah_flags & IB_AH_GRH))
420 		return -EINVAL;
421 
422 	if (ah_attr->grh.sgid_attr) {
423 		/*
424 		 * Make sure the passed sgid_attr is consistent with the
425 		 * parameters
426 		 */
427 		if (ah_attr->grh.sgid_attr->index != ah_attr->grh.sgid_index ||
428 		    ah_attr->grh.sgid_attr->port_num != ah_attr->port_num)
429 			return -EINVAL;
430 	}
431 	return 0;
432 }
433 
434 /*
435  * If the ah requires a GRH then ensure that sgid_attr pointer is filled in.
436  * On success the caller is responsible to call rdma_unfill_sgid_attr().
437  */
rdma_fill_sgid_attr(struct ib_device * device,struct rdma_ah_attr * ah_attr,const struct ib_gid_attr ** old_sgid_attr)438 static int rdma_fill_sgid_attr(struct ib_device *device,
439 			       struct rdma_ah_attr *ah_attr,
440 			       const struct ib_gid_attr **old_sgid_attr)
441 {
442 	const struct ib_gid_attr *sgid_attr;
443 	struct ib_global_route *grh;
444 	int ret;
445 
446 	*old_sgid_attr = ah_attr->grh.sgid_attr;
447 
448 	ret = rdma_check_ah_attr(device, ah_attr);
449 	if (ret)
450 		return ret;
451 
452 	if (!(ah_attr->ah_flags & IB_AH_GRH))
453 		return 0;
454 
455 	grh = rdma_ah_retrieve_grh(ah_attr);
456 	if (grh->sgid_attr)
457 		return 0;
458 
459 	sgid_attr =
460 		rdma_get_gid_attr(device, ah_attr->port_num, grh->sgid_index);
461 	if (IS_ERR(sgid_attr))
462 		return PTR_ERR(sgid_attr);
463 
464 	/* Move ownerhip of the kref into the ah_attr */
465 	grh->sgid_attr = sgid_attr;
466 	return 0;
467 }
468 
rdma_unfill_sgid_attr(struct rdma_ah_attr * ah_attr,const struct ib_gid_attr * old_sgid_attr)469 static void rdma_unfill_sgid_attr(struct rdma_ah_attr *ah_attr,
470 				  const struct ib_gid_attr *old_sgid_attr)
471 {
472 	/*
473 	 * Fill didn't change anything, the caller retains ownership of
474 	 * whatever it passed
475 	 */
476 	if (ah_attr->grh.sgid_attr == old_sgid_attr)
477 		return;
478 
479 	/*
480 	 * Otherwise, we need to undo what rdma_fill_sgid_attr so the caller
481 	 * doesn't see any change in the rdma_ah_attr. If we get here
482 	 * old_sgid_attr is NULL.
483 	 */
484 	rdma_destroy_ah_attr(ah_attr);
485 }
486 
487 static const struct ib_gid_attr *
rdma_update_sgid_attr(struct rdma_ah_attr * ah_attr,const struct ib_gid_attr * old_attr)488 rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
489 		      const struct ib_gid_attr *old_attr)
490 {
491 	if (old_attr)
492 		rdma_put_gid_attr(old_attr);
493 	if (ah_attr->ah_flags & IB_AH_GRH) {
494 		rdma_hold_gid_attr(ah_attr->grh.sgid_attr);
495 		return ah_attr->grh.sgid_attr;
496 	}
497 	return NULL;
498 }
499 
_rdma_create_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,u32 flags,struct ib_udata * udata,struct net_device * xmit_slave)500 static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
501 				     struct rdma_ah_attr *ah_attr,
502 				     u32 flags,
503 				     struct ib_udata *udata,
504 				     struct net_device *xmit_slave)
505 {
506 	struct rdma_ah_init_attr init_attr = {};
507 	struct ib_device *device = pd->device;
508 	struct ib_ah *ah;
509 	int ret;
510 
511 	might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
512 
513 	if (!udata && !device->ops.create_ah)
514 		return ERR_PTR(-EOPNOTSUPP);
515 
516 	ah = rdma_zalloc_drv_obj_gfp(
517 		device, ib_ah,
518 		(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
519 	if (!ah)
520 		return ERR_PTR(-ENOMEM);
521 
522 	ah->device = device;
523 	ah->pd = pd;
524 	ah->type = ah_attr->type;
525 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
526 	init_attr.ah_attr = ah_attr;
527 	init_attr.flags = flags;
528 	init_attr.xmit_slave = xmit_slave;
529 
530 	if (udata)
531 		ret = device->ops.create_user_ah(ah, &init_attr, udata);
532 	else
533 		ret = device->ops.create_ah(ah, &init_attr, NULL);
534 	if (ret) {
535 		if (ah->sgid_attr)
536 			rdma_put_gid_attr(ah->sgid_attr);
537 		kfree(ah);
538 		return ERR_PTR(ret);
539 	}
540 
541 	atomic_inc(&pd->usecnt);
542 	return ah;
543 }
544 
545 /**
546  * rdma_create_ah - Creates an address handle for the
547  * given address vector.
548  * @pd: The protection domain associated with the address handle.
549  * @ah_attr: The attributes of the address vector.
550  * @flags: Create address handle flags (see enum rdma_create_ah_flags).
551  *
552  * It returns 0 on success and returns appropriate error code on error.
553  * The address handle is used to reference a local or global destination
554  * in all UD QP post sends.
555  */
rdma_create_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,u32 flags)556 struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
557 			     u32 flags)
558 {
559 	const struct ib_gid_attr *old_sgid_attr;
560 	struct net_device *slave;
561 	struct ib_ah *ah;
562 	int ret;
563 
564 	ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
565 	if (ret)
566 		return ERR_PTR(ret);
567 	slave = rdma_lag_get_ah_roce_slave(pd->device, ah_attr,
568 					   (flags & RDMA_CREATE_AH_SLEEPABLE) ?
569 					   GFP_KERNEL : GFP_ATOMIC);
570 	if (IS_ERR(slave)) {
571 		rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
572 		return (void *)slave;
573 	}
574 	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);
575 	rdma_lag_put_ah_roce_slave(slave);
576 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
577 	return ah;
578 }
579 EXPORT_SYMBOL(rdma_create_ah);
580 
581 /**
582  * rdma_create_user_ah - Creates an address handle for the
583  * given address vector.
584  * It resolves destination mac address for ah attribute of RoCE type.
585  * @pd: The protection domain associated with the address handle.
586  * @ah_attr: The attributes of the address vector.
587  * @udata: pointer to user's input output buffer information need by
588  *         provider driver.
589  *
590  * It returns 0 on success and returns appropriate error code on error.
591  * The address handle is used to reference a local or global destination
592  * in all UD QP post sends.
593  */
rdma_create_user_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,struct ib_udata * udata)594 struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
595 				  struct rdma_ah_attr *ah_attr,
596 				  struct ib_udata *udata)
597 {
598 	const struct ib_gid_attr *old_sgid_attr;
599 	struct ib_ah *ah;
600 	int err;
601 
602 	err = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
603 	if (err)
604 		return ERR_PTR(err);
605 
606 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
607 		err = ib_resolve_eth_dmac(pd->device, ah_attr);
608 		if (err) {
609 			ah = ERR_PTR(err);
610 			goto out;
611 		}
612 	}
613 
614 	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,
615 			     udata, NULL);
616 
617 out:
618 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
619 	return ah;
620 }
621 EXPORT_SYMBOL(rdma_create_user_ah);
622 
ib_get_rdma_header_version(const union rdma_network_hdr * hdr)623 int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
624 {
625 	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
626 	struct iphdr ip4h_checked;
627 	const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
628 
629 	/* If it's IPv6, the version must be 6, otherwise, the first
630 	 * 20 bytes (before the IPv4 header) are garbled.
631 	 */
632 	if (ip6h->version != 6)
633 		return (ip4h->version == 4) ? 4 : 0;
634 	/* version may be 6 or 4 because the first 20 bytes could be garbled */
635 
636 	/* RoCE v2 requires no options, thus header length
637 	 * must be 5 words
638 	 */
639 	if (ip4h->ihl != 5)
640 		return 6;
641 
642 	/* Verify checksum.
643 	 * We can't write on scattered buffers so we need to copy to
644 	 * temp buffer.
645 	 */
646 	memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
647 	ip4h_checked.check = 0;
648 	ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5);
649 	/* if IPv4 header checksum is OK, believe it */
650 	if (ip4h->check == ip4h_checked.check)
651 		return 4;
652 	return 6;
653 }
654 EXPORT_SYMBOL(ib_get_rdma_header_version);
655 
ib_get_net_type_by_grh(struct ib_device * device,u32 port_num,const struct ib_grh * grh)656 static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
657 						     u32 port_num,
658 						     const struct ib_grh *grh)
659 {
660 	int grh_version;
661 
662 	if (rdma_protocol_ib(device, port_num))
663 		return RDMA_NETWORK_IB;
664 
665 	grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
666 
667 	if (grh_version == 4)
668 		return RDMA_NETWORK_IPV4;
669 
670 	if (grh->next_hdr == IPPROTO_UDP)
671 		return RDMA_NETWORK_IPV6;
672 
673 	return RDMA_NETWORK_ROCE_V1;
674 }
675 
676 struct find_gid_index_context {
677 	u16 vlan_id;
678 	enum ib_gid_type gid_type;
679 };
680 
find_gid_index(const union ib_gid * gid,const struct ib_gid_attr * gid_attr,void * context)681 static bool find_gid_index(const union ib_gid *gid,
682 			   const struct ib_gid_attr *gid_attr,
683 			   void *context)
684 {
685 	struct find_gid_index_context *ctx = context;
686 	u16 vlan_id = 0xffff;
687 	int ret;
688 
689 	if (ctx->gid_type != gid_attr->gid_type)
690 		return false;
691 
692 	ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
693 	if (ret)
694 		return false;
695 
696 	return ctx->vlan_id == vlan_id;
697 }
698 
699 static const struct ib_gid_attr *
get_sgid_attr_from_eth(struct ib_device * device,u32 port_num,u16 vlan_id,const union ib_gid * sgid,enum ib_gid_type gid_type)700 get_sgid_attr_from_eth(struct ib_device *device, u32 port_num,
701 		       u16 vlan_id, const union ib_gid *sgid,
702 		       enum ib_gid_type gid_type)
703 {
704 	struct find_gid_index_context context = {.vlan_id = vlan_id,
705 						 .gid_type = gid_type};
706 
707 	return rdma_find_gid_by_filter(device, sgid, port_num, find_gid_index,
708 				       &context);
709 }
710 
ib_get_gids_from_rdma_hdr(const union rdma_network_hdr * hdr,enum rdma_network_type net_type,union ib_gid * sgid,union ib_gid * dgid)711 int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
712 			      enum rdma_network_type net_type,
713 			      union ib_gid *sgid, union ib_gid *dgid)
714 {
715 	struct sockaddr_in  src_in;
716 	struct sockaddr_in  dst_in;
717 	__be32 src_saddr, dst_saddr;
718 
719 	if (!sgid || !dgid)
720 		return -EINVAL;
721 
722 	if (net_type == RDMA_NETWORK_IPV4) {
723 		memcpy(&src_in.sin_addr.s_addr,
724 		       &hdr->roce4grh.saddr, 4);
725 		memcpy(&dst_in.sin_addr.s_addr,
726 		       &hdr->roce4grh.daddr, 4);
727 		src_saddr = src_in.sin_addr.s_addr;
728 		dst_saddr = dst_in.sin_addr.s_addr;
729 		ipv6_addr_set_v4mapped(src_saddr,
730 				       (struct in6_addr *)sgid);
731 		ipv6_addr_set_v4mapped(dst_saddr,
732 				       (struct in6_addr *)dgid);
733 		return 0;
734 	} else if (net_type == RDMA_NETWORK_IPV6 ||
735 		   net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) {
736 		*dgid = hdr->ibgrh.dgid;
737 		*sgid = hdr->ibgrh.sgid;
738 		return 0;
739 	} else {
740 		return -EINVAL;
741 	}
742 }
743 EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
744 
745 /* Resolve destination mac address and hop limit for unicast destination
746  * GID entry, considering the source GID entry as well.
747  * ah_attribute must have have valid port_num, sgid_index.
748  */
ib_resolve_unicast_gid_dmac(struct ib_device * device,struct rdma_ah_attr * ah_attr)749 static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
750 				       struct rdma_ah_attr *ah_attr)
751 {
752 	struct ib_global_route *grh = rdma_ah_retrieve_grh(ah_attr);
753 	const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
754 	int hop_limit = 0xff;
755 	int ret = 0;
756 
757 	/* If destination is link local and source GID is RoCEv1,
758 	 * IP stack is not used.
759 	 */
760 	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) &&
761 	    sgid_attr->gid_type == IB_GID_TYPE_ROCE) {
762 		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
763 				ah_attr->roce.dmac);
764 		return ret;
765 	}
766 
767 	ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
768 					   ah_attr->roce.dmac,
769 					   sgid_attr, &hop_limit);
770 
771 	grh->hop_limit = hop_limit;
772 	return ret;
773 }
774 
775 /*
776  * This function initializes address handle attributes from the incoming packet.
777  * Incoming packet has dgid of the receiver node on which this code is
778  * getting executed and, sgid contains the GID of the sender.
779  *
780  * When resolving mac address of destination, the arrived dgid is used
781  * as sgid and, sgid is used as dgid because sgid contains destinations
782  * GID whom to respond to.
783  *
784  * On success the caller is responsible to call rdma_destroy_ah_attr on the
785  * attr.
786  */
ib_init_ah_attr_from_wc(struct ib_device * device,u32 port_num,const struct ib_wc * wc,const struct ib_grh * grh,struct rdma_ah_attr * ah_attr)787 int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
788 			    const struct ib_wc *wc, const struct ib_grh *grh,
789 			    struct rdma_ah_attr *ah_attr)
790 {
791 	u32 flow_class;
792 	int ret;
793 	enum rdma_network_type net_type = RDMA_NETWORK_IB;
794 	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
795 	const struct ib_gid_attr *sgid_attr;
796 	int hoplimit = 0xff;
797 	union ib_gid dgid;
798 	union ib_gid sgid;
799 
800 	might_sleep();
801 
802 	memset(ah_attr, 0, sizeof *ah_attr);
803 	ah_attr->type = rdma_ah_find_type(device, port_num);
804 	if (rdma_cap_eth_ah(device, port_num)) {
805 		if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
806 			net_type = wc->network_hdr_type;
807 		else
808 			net_type = ib_get_net_type_by_grh(device, port_num, grh);
809 		gid_type = ib_network_to_gid_type(net_type);
810 	}
811 	ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
812 					&sgid, &dgid);
813 	if (ret)
814 		return ret;
815 
816 	rdma_ah_set_sl(ah_attr, wc->sl);
817 	rdma_ah_set_port_num(ah_attr, port_num);
818 
819 	if (rdma_protocol_roce(device, port_num)) {
820 		u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
821 				wc->vlan_id : 0xffff;
822 
823 		if (!(wc->wc_flags & IB_WC_GRH))
824 			return -EPROTOTYPE;
825 
826 		sgid_attr = get_sgid_attr_from_eth(device, port_num,
827 						   vlan_id, &dgid,
828 						   gid_type);
829 		if (IS_ERR(sgid_attr))
830 			return PTR_ERR(sgid_attr);
831 
832 		flow_class = be32_to_cpu(grh->version_tclass_flow);
833 		rdma_move_grh_sgid_attr(ah_attr,
834 					&sgid,
835 					flow_class & 0xFFFFF,
836 					hoplimit,
837 					(flow_class >> 20) & 0xFF,
838 					sgid_attr);
839 
840 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
841 		if (ret)
842 			rdma_destroy_ah_attr(ah_attr);
843 
844 		return ret;
845 	} else {
846 		rdma_ah_set_dlid(ah_attr, wc->slid);
847 		rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
848 
849 		if ((wc->wc_flags & IB_WC_GRH) == 0)
850 			return 0;
851 
852 		if (dgid.global.interface_id !=
853 					cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
854 			sgid_attr = rdma_find_gid_by_port(
855 				device, &dgid, IB_GID_TYPE_IB, port_num, NULL);
856 		} else
857 			sgid_attr = rdma_get_gid_attr(device, port_num, 0);
858 
859 		if (IS_ERR(sgid_attr))
860 			return PTR_ERR(sgid_attr);
861 		flow_class = be32_to_cpu(grh->version_tclass_flow);
862 		rdma_move_grh_sgid_attr(ah_attr,
863 					&sgid,
864 					flow_class & 0xFFFFF,
865 					hoplimit,
866 					(flow_class >> 20) & 0xFF,
867 					sgid_attr);
868 
869 		return 0;
870 	}
871 }
872 EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
873 
874 /**
875  * rdma_move_grh_sgid_attr - Sets the sgid attribute of GRH, taking ownership
876  * of the reference
877  *
878  * @attr:	Pointer to AH attribute structure
879  * @dgid:	Destination GID
880  * @flow_label:	Flow label
881  * @hop_limit:	Hop limit
882  * @traffic_class: traffic class
883  * @sgid_attr:	Pointer to SGID attribute
884  *
885  * This takes ownership of the sgid_attr reference. The caller must ensure
886  * rdma_destroy_ah_attr() is called before destroying the rdma_ah_attr after
887  * calling this function.
888  */
rdma_move_grh_sgid_attr(struct rdma_ah_attr * attr,union ib_gid * dgid,u32 flow_label,u8 hop_limit,u8 traffic_class,const struct ib_gid_attr * sgid_attr)889 void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
890 			     u32 flow_label, u8 hop_limit, u8 traffic_class,
891 			     const struct ib_gid_attr *sgid_attr)
892 {
893 	rdma_ah_set_grh(attr, dgid, flow_label, sgid_attr->index, hop_limit,
894 			traffic_class);
895 	attr->grh.sgid_attr = sgid_attr;
896 }
897 EXPORT_SYMBOL(rdma_move_grh_sgid_attr);
898 
899 /**
900  * rdma_destroy_ah_attr - Release reference to SGID attribute of
901  * ah attribute.
902  * @ah_attr: Pointer to ah attribute
903  *
904  * Release reference to the SGID attribute of the ah attribute if it is
905  * non NULL. It is safe to call this multiple times, and safe to call it on
906  * a zero initialized ah_attr.
907  */
rdma_destroy_ah_attr(struct rdma_ah_attr * ah_attr)908 void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)
909 {
910 	if (ah_attr->grh.sgid_attr) {
911 		rdma_put_gid_attr(ah_attr->grh.sgid_attr);
912 		ah_attr->grh.sgid_attr = NULL;
913 	}
914 }
915 EXPORT_SYMBOL(rdma_destroy_ah_attr);
916 
ib_create_ah_from_wc(struct ib_pd * pd,const struct ib_wc * wc,const struct ib_grh * grh,u32 port_num)917 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
918 				   const struct ib_grh *grh, u32 port_num)
919 {
920 	struct rdma_ah_attr ah_attr;
921 	struct ib_ah *ah;
922 	int ret;
923 
924 	ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
925 	if (ret)
926 		return ERR_PTR(ret);
927 
928 	ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
929 
930 	rdma_destroy_ah_attr(&ah_attr);
931 	return ah;
932 }
933 EXPORT_SYMBOL(ib_create_ah_from_wc);
934 
rdma_modify_ah(struct ib_ah * ah,struct rdma_ah_attr * ah_attr)935 int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
936 {
937 	const struct ib_gid_attr *old_sgid_attr;
938 	int ret;
939 
940 	if (ah->type != ah_attr->type)
941 		return -EINVAL;
942 
943 	ret = rdma_fill_sgid_attr(ah->device, ah_attr, &old_sgid_attr);
944 	if (ret)
945 		return ret;
946 
947 	ret = ah->device->ops.modify_ah ?
948 		ah->device->ops.modify_ah(ah, ah_attr) :
949 		-EOPNOTSUPP;
950 
951 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
952 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
953 	return ret;
954 }
955 EXPORT_SYMBOL(rdma_modify_ah);
956 
rdma_query_ah(struct ib_ah * ah,struct rdma_ah_attr * ah_attr)957 int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
958 {
959 	ah_attr->grh.sgid_attr = NULL;
960 
961 	return ah->device->ops.query_ah ?
962 		ah->device->ops.query_ah(ah, ah_attr) :
963 		-EOPNOTSUPP;
964 }
965 EXPORT_SYMBOL(rdma_query_ah);
966 
rdma_destroy_ah_user(struct ib_ah * ah,u32 flags,struct ib_udata * udata)967 int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
968 {
969 	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
970 	struct ib_pd *pd;
971 	int ret;
972 
973 	might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
974 
975 	pd = ah->pd;
976 
977 	ret = ah->device->ops.destroy_ah(ah, flags);
978 	if (ret)
979 		return ret;
980 
981 	atomic_dec(&pd->usecnt);
982 	if (sgid_attr)
983 		rdma_put_gid_attr(sgid_attr);
984 
985 	kfree(ah);
986 	return ret;
987 }
988 EXPORT_SYMBOL(rdma_destroy_ah_user);
989 
990 /* Shared receive queues */
991 
992 /**
993  * ib_create_srq_user - Creates a SRQ associated with the specified protection
994  *   domain.
995  * @pd: The protection domain associated with the SRQ.
996  * @srq_init_attr: A list of initial attributes required to create the
997  *   SRQ.  If SRQ creation succeeds, then the attributes are updated to
998  *   the actual capabilities of the created SRQ.
999  * @uobject: uobject pointer if this is not a kernel SRQ
1000  * @udata: udata pointer if this is not a kernel SRQ
1001  *
1002  * srq_attr->max_wr and srq_attr->max_sge are read the determine the
1003  * requested size of the SRQ, and set to the actual values allocated
1004  * on return.  If ib_create_srq() succeeds, then max_wr and max_sge
1005  * will always be at least as large as the requested values.
1006  */
ib_create_srq_user(struct ib_pd * pd,struct ib_srq_init_attr * srq_init_attr,struct ib_usrq_object * uobject,struct ib_udata * udata)1007 struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
1008 				  struct ib_srq_init_attr *srq_init_attr,
1009 				  struct ib_usrq_object *uobject,
1010 				  struct ib_udata *udata)
1011 {
1012 	struct ib_srq *srq;
1013 	int ret;
1014 
1015 	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
1016 	if (!srq)
1017 		return ERR_PTR(-ENOMEM);
1018 
1019 	srq->device = pd->device;
1020 	srq->pd = pd;
1021 	srq->event_handler = srq_init_attr->event_handler;
1022 	srq->srq_context = srq_init_attr->srq_context;
1023 	srq->srq_type = srq_init_attr->srq_type;
1024 	srq->uobject = uobject;
1025 
1026 	if (ib_srq_has_cq(srq->srq_type)) {
1027 		srq->ext.cq = srq_init_attr->ext.cq;
1028 		atomic_inc(&srq->ext.cq->usecnt);
1029 	}
1030 	if (srq->srq_type == IB_SRQT_XRC) {
1031 		srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
1032 		if (srq->ext.xrc.xrcd)
1033 			atomic_inc(&srq->ext.xrc.xrcd->usecnt);
1034 	}
1035 	atomic_inc(&pd->usecnt);
1036 
1037 	rdma_restrack_new(&srq->res, RDMA_RESTRACK_SRQ);
1038 	rdma_restrack_parent_name(&srq->res, &pd->res);
1039 
1040 	ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
1041 	if (ret) {
1042 		rdma_restrack_put(&srq->res);
1043 		atomic_dec(&pd->usecnt);
1044 		if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1045 			atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1046 		if (ib_srq_has_cq(srq->srq_type))
1047 			atomic_dec(&srq->ext.cq->usecnt);
1048 		kfree(srq);
1049 		return ERR_PTR(ret);
1050 	}
1051 
1052 	rdma_restrack_add(&srq->res);
1053 
1054 	return srq;
1055 }
1056 EXPORT_SYMBOL(ib_create_srq_user);
1057 
ib_modify_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask)1058 int ib_modify_srq(struct ib_srq *srq,
1059 		  struct ib_srq_attr *srq_attr,
1060 		  enum ib_srq_attr_mask srq_attr_mask)
1061 {
1062 	return srq->device->ops.modify_srq ?
1063 		srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
1064 					    NULL) : -EOPNOTSUPP;
1065 }
1066 EXPORT_SYMBOL(ib_modify_srq);
1067 
ib_query_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr)1068 int ib_query_srq(struct ib_srq *srq,
1069 		 struct ib_srq_attr *srq_attr)
1070 {
1071 	return srq->device->ops.query_srq ?
1072 		srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
1073 }
1074 EXPORT_SYMBOL(ib_query_srq);
1075 
ib_destroy_srq_user(struct ib_srq * srq,struct ib_udata * udata)1076 int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
1077 {
1078 	int ret;
1079 
1080 	if (atomic_read(&srq->usecnt))
1081 		return -EBUSY;
1082 
1083 	ret = srq->device->ops.destroy_srq(srq, udata);
1084 	if (ret)
1085 		return ret;
1086 
1087 	atomic_dec(&srq->pd->usecnt);
1088 	if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1089 		atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1090 	if (ib_srq_has_cq(srq->srq_type))
1091 		atomic_dec(&srq->ext.cq->usecnt);
1092 	rdma_restrack_del(&srq->res);
1093 	kfree(srq);
1094 
1095 	return ret;
1096 }
1097 EXPORT_SYMBOL(ib_destroy_srq_user);
1098 
1099 /* Queue pairs */
1100 
__ib_shared_qp_event_handler(struct ib_event * event,void * context)1101 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
1102 {
1103 	struct ib_qp *qp = context;
1104 	unsigned long flags;
1105 
1106 	spin_lock_irqsave(&qp->device->qp_open_list_lock, flags);
1107 	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1108 		if (event->element.qp->event_handler)
1109 			event->element.qp->event_handler(event, event->element.qp->qp_context);
1110 	spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
1111 }
1112 
__ib_open_qp(struct ib_qp * real_qp,void (* event_handler)(struct ib_event *,void *),void * qp_context)1113 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
1114 				  void (*event_handler)(struct ib_event *, void *),
1115 				  void *qp_context)
1116 {
1117 	struct ib_qp *qp;
1118 	unsigned long flags;
1119 	int err;
1120 
1121 	qp = kzalloc(sizeof *qp, GFP_KERNEL);
1122 	if (!qp)
1123 		return ERR_PTR(-ENOMEM);
1124 
1125 	qp->real_qp = real_qp;
1126 	err = ib_open_shared_qp_security(qp, real_qp->device);
1127 	if (err) {
1128 		kfree(qp);
1129 		return ERR_PTR(err);
1130 	}
1131 
1132 	qp->real_qp = real_qp;
1133 	atomic_inc(&real_qp->usecnt);
1134 	qp->device = real_qp->device;
1135 	qp->event_handler = event_handler;
1136 	qp->qp_context = qp_context;
1137 	qp->qp_num = real_qp->qp_num;
1138 	qp->qp_type = real_qp->qp_type;
1139 
1140 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1141 	list_add(&qp->open_list, &real_qp->open_list);
1142 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1143 
1144 	return qp;
1145 }
1146 
ib_open_qp(struct ib_xrcd * xrcd,struct ib_qp_open_attr * qp_open_attr)1147 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
1148 			 struct ib_qp_open_attr *qp_open_attr)
1149 {
1150 	struct ib_qp *qp, *real_qp;
1151 
1152 	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1153 		return ERR_PTR(-EINVAL);
1154 
1155 	down_read(&xrcd->tgt_qps_rwsem);
1156 	real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
1157 	if (!real_qp) {
1158 		up_read(&xrcd->tgt_qps_rwsem);
1159 		return ERR_PTR(-EINVAL);
1160 	}
1161 	qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
1162 			  qp_open_attr->qp_context);
1163 	up_read(&xrcd->tgt_qps_rwsem);
1164 	return qp;
1165 }
1166 EXPORT_SYMBOL(ib_open_qp);
1167 
create_xrc_qp_user(struct ib_qp * qp,struct ib_qp_init_attr * qp_init_attr)1168 static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
1169 					struct ib_qp_init_attr *qp_init_attr)
1170 {
1171 	struct ib_qp *real_qp = qp;
1172 	int err;
1173 
1174 	qp->event_handler = __ib_shared_qp_event_handler;
1175 	qp->qp_context = qp;
1176 	qp->pd = NULL;
1177 	qp->send_cq = qp->recv_cq = NULL;
1178 	qp->srq = NULL;
1179 	qp->xrcd = qp_init_attr->xrcd;
1180 	atomic_inc(&qp_init_attr->xrcd->usecnt);
1181 	INIT_LIST_HEAD(&qp->open_list);
1182 
1183 	qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
1184 			  qp_init_attr->qp_context);
1185 	if (IS_ERR(qp))
1186 		return qp;
1187 
1188 	err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
1189 			      real_qp, GFP_KERNEL));
1190 	if (err) {
1191 		ib_close_qp(qp);
1192 		return ERR_PTR(err);
1193 	}
1194 	return qp;
1195 }
1196 
create_qp(struct ib_device * dev,struct ib_pd * pd,struct ib_qp_init_attr * attr,struct ib_udata * udata,struct ib_uqp_object * uobj,const char * caller)1197 static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
1198 			       struct ib_qp_init_attr *attr,
1199 			       struct ib_udata *udata,
1200 			       struct ib_uqp_object *uobj, const char *caller)
1201 {
1202 	struct ib_udata dummy = {};
1203 	struct ib_qp *qp;
1204 	int ret;
1205 
1206 	if (!dev->ops.create_qp)
1207 		return ERR_PTR(-EOPNOTSUPP);
1208 
1209 	qp = rdma_zalloc_drv_obj_numa(dev, ib_qp);
1210 	if (!qp)
1211 		return ERR_PTR(-ENOMEM);
1212 
1213 	qp->device = dev;
1214 	qp->pd = pd;
1215 	qp->uobject = uobj;
1216 	qp->real_qp = qp;
1217 
1218 	qp->qp_type = attr->qp_type;
1219 	qp->rwq_ind_tbl = attr->rwq_ind_tbl;
1220 	qp->srq = attr->srq;
1221 	qp->event_handler = attr->event_handler;
1222 	qp->port = attr->port_num;
1223 	qp->qp_context = attr->qp_context;
1224 
1225 	spin_lock_init(&qp->mr_lock);
1226 	INIT_LIST_HEAD(&qp->rdma_mrs);
1227 	INIT_LIST_HEAD(&qp->sig_mrs);
1228 
1229 	qp->send_cq = attr->send_cq;
1230 	qp->recv_cq = attr->recv_cq;
1231 
1232 	rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
1233 	WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
1234 	rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
1235 	ret = dev->ops.create_qp(qp, attr, udata);
1236 	if (ret)
1237 		goto err_create;
1238 
1239 	/*
1240 	 * TODO: The mlx4 internally overwrites send_cq and recv_cq.
1241 	 * Unfortunately, it is not an easy task to fix that driver.
1242 	 */
1243 	qp->send_cq = attr->send_cq;
1244 	qp->recv_cq = attr->recv_cq;
1245 
1246 	ret = ib_create_qp_security(qp, dev);
1247 	if (ret)
1248 		goto err_security;
1249 
1250 	rdma_restrack_add(&qp->res);
1251 	return qp;
1252 
1253 err_security:
1254 	qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL);
1255 err_create:
1256 	rdma_restrack_put(&qp->res);
1257 	kfree(qp);
1258 	return ERR_PTR(ret);
1259 
1260 }
1261 
1262 /**
1263  * ib_create_qp_user - Creates a QP associated with the specified protection
1264  *   domain.
1265  * @dev: IB device
1266  * @pd: The protection domain associated with the QP.
1267  * @attr: A list of initial attributes required to create the
1268  *   QP.  If QP creation succeeds, then the attributes are updated to
1269  *   the actual capabilities of the created QP.
1270  * @udata: User data
1271  * @uobj: uverbs obect
1272  * @caller: caller's build-time module name
1273  */
ib_create_qp_user(struct ib_device * dev,struct ib_pd * pd,struct ib_qp_init_attr * attr,struct ib_udata * udata,struct ib_uqp_object * uobj,const char * caller)1274 struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
1275 				struct ib_qp_init_attr *attr,
1276 				struct ib_udata *udata,
1277 				struct ib_uqp_object *uobj, const char *caller)
1278 {
1279 	struct ib_qp *qp, *xrc_qp;
1280 
1281 	if (attr->qp_type == IB_QPT_XRC_TGT)
1282 		qp = create_qp(dev, pd, attr, NULL, NULL, caller);
1283 	else
1284 		qp = create_qp(dev, pd, attr, udata, uobj, NULL);
1285 	if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp))
1286 		return qp;
1287 
1288 	xrc_qp = create_xrc_qp_user(qp, attr);
1289 	if (IS_ERR(xrc_qp)) {
1290 		ib_destroy_qp(qp);
1291 		return xrc_qp;
1292 	}
1293 
1294 	xrc_qp->uobject = uobj;
1295 	return xrc_qp;
1296 }
1297 EXPORT_SYMBOL(ib_create_qp_user);
1298 
ib_qp_usecnt_inc(struct ib_qp * qp)1299 void ib_qp_usecnt_inc(struct ib_qp *qp)
1300 {
1301 	if (qp->pd)
1302 		atomic_inc(&qp->pd->usecnt);
1303 	if (qp->send_cq)
1304 		atomic_inc(&qp->send_cq->usecnt);
1305 	if (qp->recv_cq)
1306 		atomic_inc(&qp->recv_cq->usecnt);
1307 	if (qp->srq)
1308 		atomic_inc(&qp->srq->usecnt);
1309 	if (qp->rwq_ind_tbl)
1310 		atomic_inc(&qp->rwq_ind_tbl->usecnt);
1311 }
1312 EXPORT_SYMBOL(ib_qp_usecnt_inc);
1313 
ib_qp_usecnt_dec(struct ib_qp * qp)1314 void ib_qp_usecnt_dec(struct ib_qp *qp)
1315 {
1316 	if (qp->rwq_ind_tbl)
1317 		atomic_dec(&qp->rwq_ind_tbl->usecnt);
1318 	if (qp->srq)
1319 		atomic_dec(&qp->srq->usecnt);
1320 	if (qp->recv_cq)
1321 		atomic_dec(&qp->recv_cq->usecnt);
1322 	if (qp->send_cq)
1323 		atomic_dec(&qp->send_cq->usecnt);
1324 	if (qp->pd)
1325 		atomic_dec(&qp->pd->usecnt);
1326 }
1327 EXPORT_SYMBOL(ib_qp_usecnt_dec);
1328 
ib_create_qp_kernel(struct ib_pd * pd,struct ib_qp_init_attr * qp_init_attr,const char * caller)1329 struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
1330 				  struct ib_qp_init_attr *qp_init_attr,
1331 				  const char *caller)
1332 {
1333 	struct ib_device *device = pd->device;
1334 	struct ib_qp *qp;
1335 	int ret;
1336 
1337 	/*
1338 	 * If the callers is using the RDMA API calculate the resources
1339 	 * needed for the RDMA READ/WRITE operations.
1340 	 *
1341 	 * Note that these callers need to pass in a port number.
1342 	 */
1343 	if (qp_init_attr->cap.max_rdma_ctxs)
1344 		rdma_rw_init_qp(device, qp_init_attr);
1345 
1346 	qp = create_qp(device, pd, qp_init_attr, NULL, NULL, caller);
1347 	if (IS_ERR(qp))
1348 		return qp;
1349 
1350 	ib_qp_usecnt_inc(qp);
1351 
1352 	if (qp_init_attr->cap.max_rdma_ctxs) {
1353 		ret = rdma_rw_init_mrs(qp, qp_init_attr);
1354 		if (ret)
1355 			goto err;
1356 	}
1357 
1358 	/*
1359 	 * Note: all hw drivers guarantee that max_send_sge is lower than
1360 	 * the device RDMA WRITE SGE limit but not all hw drivers ensure that
1361 	 * max_send_sge <= max_sge_rd.
1362 	 */
1363 	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1364 	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1365 				 device->attrs.max_sge_rd);
1366 	if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
1367 		qp->integrity_en = true;
1368 
1369 	return qp;
1370 
1371 err:
1372 	ib_destroy_qp(qp);
1373 	return ERR_PTR(ret);
1374 
1375 }
1376 EXPORT_SYMBOL(ib_create_qp_kernel);
1377 
1378 static const struct {
1379 	int			valid;
1380 	enum ib_qp_attr_mask	req_param[IB_QPT_MAX];
1381 	enum ib_qp_attr_mask	opt_param[IB_QPT_MAX];
1382 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
1383 	[IB_QPS_RESET] = {
1384 		[IB_QPS_RESET] = { .valid = 1 },
1385 		[IB_QPS_INIT]  = {
1386 			.valid = 1,
1387 			.req_param = {
1388 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1389 						IB_QP_PORT			|
1390 						IB_QP_QKEY),
1391 				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
1392 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1393 						IB_QP_PORT			|
1394 						IB_QP_ACCESS_FLAGS),
1395 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1396 						IB_QP_PORT			|
1397 						IB_QP_ACCESS_FLAGS),
1398 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1399 						IB_QP_PORT			|
1400 						IB_QP_ACCESS_FLAGS),
1401 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1402 						IB_QP_PORT			|
1403 						IB_QP_ACCESS_FLAGS),
1404 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1405 						IB_QP_QKEY),
1406 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1407 						IB_QP_QKEY),
1408 			}
1409 		},
1410 	},
1411 	[IB_QPS_INIT]  = {
1412 		[IB_QPS_RESET] = { .valid = 1 },
1413 		[IB_QPS_ERR] =   { .valid = 1 },
1414 		[IB_QPS_INIT]  = {
1415 			.valid = 1,
1416 			.opt_param = {
1417 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1418 						IB_QP_PORT			|
1419 						IB_QP_QKEY),
1420 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1421 						IB_QP_PORT			|
1422 						IB_QP_ACCESS_FLAGS),
1423 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1424 						IB_QP_PORT			|
1425 						IB_QP_ACCESS_FLAGS),
1426 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1427 						IB_QP_PORT			|
1428 						IB_QP_ACCESS_FLAGS),
1429 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1430 						IB_QP_PORT			|
1431 						IB_QP_ACCESS_FLAGS),
1432 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1433 						IB_QP_QKEY),
1434 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1435 						IB_QP_QKEY),
1436 			}
1437 		},
1438 		[IB_QPS_RTR]   = {
1439 			.valid = 1,
1440 			.req_param = {
1441 				[IB_QPT_UC]  = (IB_QP_AV			|
1442 						IB_QP_PATH_MTU			|
1443 						IB_QP_DEST_QPN			|
1444 						IB_QP_RQ_PSN),
1445 				[IB_QPT_RC]  = (IB_QP_AV			|
1446 						IB_QP_PATH_MTU			|
1447 						IB_QP_DEST_QPN			|
1448 						IB_QP_RQ_PSN			|
1449 						IB_QP_MAX_DEST_RD_ATOMIC	|
1450 						IB_QP_MIN_RNR_TIMER),
1451 				[IB_QPT_XRC_INI] = (IB_QP_AV			|
1452 						IB_QP_PATH_MTU			|
1453 						IB_QP_DEST_QPN			|
1454 						IB_QP_RQ_PSN),
1455 				[IB_QPT_XRC_TGT] = (IB_QP_AV			|
1456 						IB_QP_PATH_MTU			|
1457 						IB_QP_DEST_QPN			|
1458 						IB_QP_RQ_PSN			|
1459 						IB_QP_MAX_DEST_RD_ATOMIC	|
1460 						IB_QP_MIN_RNR_TIMER),
1461 			},
1462 			.opt_param = {
1463 				 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1464 						 IB_QP_QKEY),
1465 				 [IB_QPT_UC]  = (IB_QP_ALT_PATH			|
1466 						 IB_QP_ACCESS_FLAGS		|
1467 						 IB_QP_PKEY_INDEX),
1468 				 [IB_QPT_RC]  = (IB_QP_ALT_PATH			|
1469 						 IB_QP_ACCESS_FLAGS		|
1470 						 IB_QP_PKEY_INDEX),
1471 				 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH		|
1472 						 IB_QP_ACCESS_FLAGS		|
1473 						 IB_QP_PKEY_INDEX),
1474 				 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH		|
1475 						 IB_QP_ACCESS_FLAGS		|
1476 						 IB_QP_PKEY_INDEX),
1477 				 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1478 						 IB_QP_QKEY),
1479 				 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1480 						 IB_QP_QKEY),
1481 			 },
1482 		},
1483 	},
1484 	[IB_QPS_RTR]   = {
1485 		[IB_QPS_RESET] = { .valid = 1 },
1486 		[IB_QPS_ERR] =   { .valid = 1 },
1487 		[IB_QPS_RTS]   = {
1488 			.valid = 1,
1489 			.req_param = {
1490 				[IB_QPT_UD]  = IB_QP_SQ_PSN,
1491 				[IB_QPT_UC]  = IB_QP_SQ_PSN,
1492 				[IB_QPT_RC]  = (IB_QP_TIMEOUT			|
1493 						IB_QP_RETRY_CNT			|
1494 						IB_QP_RNR_RETRY			|
1495 						IB_QP_SQ_PSN			|
1496 						IB_QP_MAX_QP_RD_ATOMIC),
1497 				[IB_QPT_XRC_INI] = (IB_QP_TIMEOUT		|
1498 						IB_QP_RETRY_CNT			|
1499 						IB_QP_RNR_RETRY			|
1500 						IB_QP_SQ_PSN			|
1501 						IB_QP_MAX_QP_RD_ATOMIC),
1502 				[IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT		|
1503 						IB_QP_SQ_PSN),
1504 				[IB_QPT_SMI] = IB_QP_SQ_PSN,
1505 				[IB_QPT_GSI] = IB_QP_SQ_PSN,
1506 			},
1507 			.opt_param = {
1508 				 [IB_QPT_UD]  = (IB_QP_CUR_STATE		|
1509 						 IB_QP_QKEY),
1510 				 [IB_QPT_UC]  = (IB_QP_CUR_STATE		|
1511 						 IB_QP_ALT_PATH			|
1512 						 IB_QP_ACCESS_FLAGS		|
1513 						 IB_QP_PATH_MIG_STATE),
1514 				 [IB_QPT_RC]  = (IB_QP_CUR_STATE		|
1515 						 IB_QP_ALT_PATH			|
1516 						 IB_QP_ACCESS_FLAGS		|
1517 						 IB_QP_MIN_RNR_TIMER		|
1518 						 IB_QP_PATH_MIG_STATE),
1519 				 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1520 						 IB_QP_ALT_PATH			|
1521 						 IB_QP_ACCESS_FLAGS		|
1522 						 IB_QP_PATH_MIG_STATE),
1523 				 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1524 						 IB_QP_ALT_PATH			|
1525 						 IB_QP_ACCESS_FLAGS		|
1526 						 IB_QP_MIN_RNR_TIMER		|
1527 						 IB_QP_PATH_MIG_STATE),
1528 				 [IB_QPT_SMI] = (IB_QP_CUR_STATE		|
1529 						 IB_QP_QKEY),
1530 				 [IB_QPT_GSI] = (IB_QP_CUR_STATE		|
1531 						 IB_QP_QKEY),
1532 				 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1533 			 }
1534 		}
1535 	},
1536 	[IB_QPS_RTS]   = {
1537 		[IB_QPS_RESET] = { .valid = 1 },
1538 		[IB_QPS_ERR] =   { .valid = 1 },
1539 		[IB_QPS_RTS]   = {
1540 			.valid = 1,
1541 			.opt_param = {
1542 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1543 						IB_QP_QKEY),
1544 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1545 						IB_QP_ACCESS_FLAGS		|
1546 						IB_QP_ALT_PATH			|
1547 						IB_QP_PATH_MIG_STATE),
1548 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1549 						IB_QP_ACCESS_FLAGS		|
1550 						IB_QP_ALT_PATH			|
1551 						IB_QP_PATH_MIG_STATE		|
1552 						IB_QP_MIN_RNR_TIMER),
1553 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1554 						IB_QP_ACCESS_FLAGS		|
1555 						IB_QP_ALT_PATH			|
1556 						IB_QP_PATH_MIG_STATE),
1557 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1558 						IB_QP_ACCESS_FLAGS		|
1559 						IB_QP_ALT_PATH			|
1560 						IB_QP_PATH_MIG_STATE		|
1561 						IB_QP_MIN_RNR_TIMER),
1562 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1563 						IB_QP_QKEY),
1564 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1565 						IB_QP_QKEY),
1566 				[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1567 			}
1568 		},
1569 		[IB_QPS_SQD]   = {
1570 			.valid = 1,
1571 			.opt_param = {
1572 				[IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1573 				[IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1574 				[IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1575 				[IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1576 				[IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
1577 				[IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1578 				[IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
1579 			}
1580 		},
1581 	},
1582 	[IB_QPS_SQD]   = {
1583 		[IB_QPS_RESET] = { .valid = 1 },
1584 		[IB_QPS_ERR] =   { .valid = 1 },
1585 		[IB_QPS_RTS]   = {
1586 			.valid = 1,
1587 			.opt_param = {
1588 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1589 						IB_QP_QKEY),
1590 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1591 						IB_QP_ALT_PATH			|
1592 						IB_QP_ACCESS_FLAGS		|
1593 						IB_QP_PATH_MIG_STATE),
1594 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1595 						IB_QP_ALT_PATH			|
1596 						IB_QP_ACCESS_FLAGS		|
1597 						IB_QP_MIN_RNR_TIMER		|
1598 						IB_QP_PATH_MIG_STATE),
1599 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1600 						IB_QP_ALT_PATH			|
1601 						IB_QP_ACCESS_FLAGS		|
1602 						IB_QP_PATH_MIG_STATE),
1603 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1604 						IB_QP_ALT_PATH			|
1605 						IB_QP_ACCESS_FLAGS		|
1606 						IB_QP_MIN_RNR_TIMER		|
1607 						IB_QP_PATH_MIG_STATE),
1608 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1609 						IB_QP_QKEY),
1610 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1611 						IB_QP_QKEY),
1612 			}
1613 		},
1614 		[IB_QPS_SQD]   = {
1615 			.valid = 1,
1616 			.opt_param = {
1617 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1618 						IB_QP_QKEY),
1619 				[IB_QPT_UC]  = (IB_QP_AV			|
1620 						IB_QP_ALT_PATH			|
1621 						IB_QP_ACCESS_FLAGS		|
1622 						IB_QP_PKEY_INDEX		|
1623 						IB_QP_PATH_MIG_STATE),
1624 				[IB_QPT_RC]  = (IB_QP_PORT			|
1625 						IB_QP_AV			|
1626 						IB_QP_TIMEOUT			|
1627 						IB_QP_RETRY_CNT			|
1628 						IB_QP_RNR_RETRY			|
1629 						IB_QP_MAX_QP_RD_ATOMIC		|
1630 						IB_QP_MAX_DEST_RD_ATOMIC	|
1631 						IB_QP_ALT_PATH			|
1632 						IB_QP_ACCESS_FLAGS		|
1633 						IB_QP_PKEY_INDEX		|
1634 						IB_QP_MIN_RNR_TIMER		|
1635 						IB_QP_PATH_MIG_STATE),
1636 				[IB_QPT_XRC_INI] = (IB_QP_PORT			|
1637 						IB_QP_AV			|
1638 						IB_QP_TIMEOUT			|
1639 						IB_QP_RETRY_CNT			|
1640 						IB_QP_RNR_RETRY			|
1641 						IB_QP_MAX_QP_RD_ATOMIC		|
1642 						IB_QP_ALT_PATH			|
1643 						IB_QP_ACCESS_FLAGS		|
1644 						IB_QP_PKEY_INDEX		|
1645 						IB_QP_PATH_MIG_STATE),
1646 				[IB_QPT_XRC_TGT] = (IB_QP_PORT			|
1647 						IB_QP_AV			|
1648 						IB_QP_TIMEOUT			|
1649 						IB_QP_MAX_DEST_RD_ATOMIC	|
1650 						IB_QP_ALT_PATH			|
1651 						IB_QP_ACCESS_FLAGS		|
1652 						IB_QP_PKEY_INDEX		|
1653 						IB_QP_MIN_RNR_TIMER		|
1654 						IB_QP_PATH_MIG_STATE),
1655 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1656 						IB_QP_QKEY),
1657 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1658 						IB_QP_QKEY),
1659 			}
1660 		}
1661 	},
1662 	[IB_QPS_SQE]   = {
1663 		[IB_QPS_RESET] = { .valid = 1 },
1664 		[IB_QPS_ERR] =   { .valid = 1 },
1665 		[IB_QPS_RTS]   = {
1666 			.valid = 1,
1667 			.opt_param = {
1668 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1669 						IB_QP_QKEY),
1670 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1671 						IB_QP_ACCESS_FLAGS),
1672 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1673 						IB_QP_QKEY),
1674 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1675 						IB_QP_QKEY),
1676 			}
1677 		}
1678 	},
1679 	[IB_QPS_ERR] = {
1680 		[IB_QPS_RESET] = { .valid = 1 },
1681 		[IB_QPS_ERR] =   { .valid = 1 }
1682 	}
1683 };
1684 
ib_modify_qp_is_ok(enum ib_qp_state cur_state,enum ib_qp_state next_state,enum ib_qp_type type,enum ib_qp_attr_mask mask)1685 bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1686 			enum ib_qp_type type, enum ib_qp_attr_mask mask)
1687 {
1688 	enum ib_qp_attr_mask req_param, opt_param;
1689 
1690 	if (mask & IB_QP_CUR_STATE  &&
1691 	    cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
1692 	    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
1693 		return false;
1694 
1695 	if (!qp_state_table[cur_state][next_state].valid)
1696 		return false;
1697 
1698 	req_param = qp_state_table[cur_state][next_state].req_param[type];
1699 	opt_param = qp_state_table[cur_state][next_state].opt_param[type];
1700 
1701 	if ((mask & req_param) != req_param)
1702 		return false;
1703 
1704 	if (mask & ~(req_param | opt_param | IB_QP_STATE))
1705 		return false;
1706 
1707 	return true;
1708 }
1709 EXPORT_SYMBOL(ib_modify_qp_is_ok);
1710 
1711 /**
1712  * ib_resolve_eth_dmac - Resolve destination mac address
1713  * @device:		Device to consider
1714  * @ah_attr:		address handle attribute which describes the
1715  *			source and destination parameters
1716  * ib_resolve_eth_dmac() resolves destination mac address and L3 hop limit It
1717  * returns 0 on success or appropriate error code. It initializes the
1718  * necessary ah_attr fields when call is successful.
1719  */
ib_resolve_eth_dmac(struct ib_device * device,struct rdma_ah_attr * ah_attr)1720 static int ib_resolve_eth_dmac(struct ib_device *device,
1721 			       struct rdma_ah_attr *ah_attr)
1722 {
1723 	int ret = 0;
1724 
1725 	if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1726 		if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1727 			__be32 addr = 0;
1728 
1729 			memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
1730 			ip_eth_mc_map(addr, (char *)ah_attr->roce.dmac);
1731 		} else {
1732 			ipv6_eth_mc_map((struct in6_addr *)ah_attr->grh.dgid.raw,
1733 					(char *)ah_attr->roce.dmac);
1734 		}
1735 	} else {
1736 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
1737 	}
1738 	return ret;
1739 }
1740 
is_qp_type_connected(const struct ib_qp * qp)1741 static bool is_qp_type_connected(const struct ib_qp *qp)
1742 {
1743 	return (qp->qp_type == IB_QPT_UC ||
1744 		qp->qp_type == IB_QPT_RC ||
1745 		qp->qp_type == IB_QPT_XRC_INI ||
1746 		qp->qp_type == IB_QPT_XRC_TGT);
1747 }
1748 
1749 /*
1750  * IB core internal function to perform QP attributes modification.
1751  */
_ib_modify_qp(struct ib_qp * qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1752 static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1753 			 int attr_mask, struct ib_udata *udata)
1754 {
1755 	u32 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1756 	const struct ib_gid_attr *old_sgid_attr_av;
1757 	const struct ib_gid_attr *old_sgid_attr_alt_av;
1758 	int ret;
1759 
1760 	attr->xmit_slave = NULL;
1761 	if (attr_mask & IB_QP_AV) {
1762 		ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
1763 					  &old_sgid_attr_av);
1764 		if (ret)
1765 			return ret;
1766 
1767 		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1768 		    is_qp_type_connected(qp)) {
1769 			struct net_device *slave;
1770 
1771 			/*
1772 			 * If the user provided the qp_attr then we have to
1773 			 * resolve it. Kerne users have to provide already
1774 			 * resolved rdma_ah_attr's.
1775 			 */
1776 			if (udata) {
1777 				ret = ib_resolve_eth_dmac(qp->device,
1778 							  &attr->ah_attr);
1779 				if (ret)
1780 					goto out_av;
1781 			}
1782 			slave = rdma_lag_get_ah_roce_slave(qp->device,
1783 							   &attr->ah_attr,
1784 							   GFP_KERNEL);
1785 			if (IS_ERR(slave)) {
1786 				ret = PTR_ERR(slave);
1787 				goto out_av;
1788 			}
1789 			attr->xmit_slave = slave;
1790 		}
1791 	}
1792 	if (attr_mask & IB_QP_ALT_PATH) {
1793 		/*
1794 		 * FIXME: This does not track the migration state, so if the
1795 		 * user loads a new alternate path after the HW has migrated
1796 		 * from primary->alternate we will keep the wrong
1797 		 * references. This is OK for IB because the reference
1798 		 * counting does not serve any functional purpose.
1799 		 */
1800 		ret = rdma_fill_sgid_attr(qp->device, &attr->alt_ah_attr,
1801 					  &old_sgid_attr_alt_av);
1802 		if (ret)
1803 			goto out_av;
1804 
1805 		/*
1806 		 * Today the core code can only handle alternate paths and APM
1807 		 * for IB. Ban them in roce mode.
1808 		 */
1809 		if (!(rdma_protocol_ib(qp->device,
1810 				       attr->alt_ah_attr.port_num) &&
1811 		      rdma_protocol_ib(qp->device, port))) {
1812 			ret = -EINVAL;
1813 			goto out;
1814 		}
1815 	}
1816 
1817 	if (rdma_ib_or_roce(qp->device, port)) {
1818 		if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1819 			dev_warn(&qp->device->dev,
1820 				 "%s rq_psn overflow, masking to 24 bits\n",
1821 				 __func__);
1822 			attr->rq_psn &= 0xffffff;
1823 		}
1824 
1825 		if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1826 			dev_warn(&qp->device->dev,
1827 				 " %s sq_psn overflow, masking to 24 bits\n",
1828 				 __func__);
1829 			attr->sq_psn &= 0xffffff;
1830 		}
1831 	}
1832 
1833 	/*
1834 	 * Bind this qp to a counter automatically based on the rdma counter
1835 	 * rules. This only set in RST2INIT with port specified
1836 	 */
1837 	if (!qp->counter && (attr_mask & IB_QP_PORT) &&
1838 	    ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
1839 		rdma_counter_bind_qp_auto(qp, attr->port_num);
1840 
1841 	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
1842 	if (ret)
1843 		goto out;
1844 
1845 	if (attr_mask & IB_QP_PORT)
1846 		qp->port = attr->port_num;
1847 	if (attr_mask & IB_QP_AV)
1848 		qp->av_sgid_attr =
1849 			rdma_update_sgid_attr(&attr->ah_attr, qp->av_sgid_attr);
1850 	if (attr_mask & IB_QP_ALT_PATH)
1851 		qp->alt_path_sgid_attr = rdma_update_sgid_attr(
1852 			&attr->alt_ah_attr, qp->alt_path_sgid_attr);
1853 
1854 out:
1855 	if (attr_mask & IB_QP_ALT_PATH)
1856 		rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
1857 out_av:
1858 	if (attr_mask & IB_QP_AV) {
1859 		rdma_lag_put_ah_roce_slave(attr->xmit_slave);
1860 		rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
1861 	}
1862 	return ret;
1863 }
1864 
1865 /**
1866  * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
1867  * @ib_qp: The QP to modify.
1868  * @attr: On input, specifies the QP attributes to modify.  On output,
1869  *   the current values of selected QP attributes are returned.
1870  * @attr_mask: A bit-mask used to specify which attributes of the QP
1871  *   are being modified.
1872  * @udata: pointer to user's input output buffer information
1873  *   are being modified.
1874  * It returns 0 on success and returns appropriate error code on error.
1875  */
ib_modify_qp_with_udata(struct ib_qp * ib_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1876 int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
1877 			    int attr_mask, struct ib_udata *udata)
1878 {
1879 	return _ib_modify_qp(ib_qp->real_qp, attr, attr_mask, udata);
1880 }
1881 EXPORT_SYMBOL(ib_modify_qp_with_udata);
1882 
ib_get_width_and_speed(u32 netdev_speed,u32 lanes,u16 * speed,u8 * width)1883 static void ib_get_width_and_speed(u32 netdev_speed, u32 lanes,
1884 				   u16 *speed, u8 *width)
1885 {
1886 	if (!lanes) {
1887 		if (netdev_speed <= SPEED_1000) {
1888 			*width = IB_WIDTH_1X;
1889 			*speed = IB_SPEED_SDR;
1890 		} else if (netdev_speed <= SPEED_10000) {
1891 			*width = IB_WIDTH_1X;
1892 			*speed = IB_SPEED_FDR10;
1893 		} else if (netdev_speed <= SPEED_20000) {
1894 			*width = IB_WIDTH_4X;
1895 			*speed = IB_SPEED_DDR;
1896 		} else if (netdev_speed <= SPEED_25000) {
1897 			*width = IB_WIDTH_1X;
1898 			*speed = IB_SPEED_EDR;
1899 		} else if (netdev_speed <= SPEED_40000) {
1900 			*width = IB_WIDTH_4X;
1901 			*speed = IB_SPEED_FDR10;
1902 		} else if (netdev_speed <= SPEED_50000) {
1903 			*width = IB_WIDTH_2X;
1904 			*speed = IB_SPEED_EDR;
1905 		} else if (netdev_speed <= SPEED_100000) {
1906 			*width = IB_WIDTH_4X;
1907 			*speed = IB_SPEED_EDR;
1908 		} else if (netdev_speed <= SPEED_200000) {
1909 			*width = IB_WIDTH_4X;
1910 			*speed = IB_SPEED_HDR;
1911 		} else {
1912 			*width = IB_WIDTH_4X;
1913 			*speed = IB_SPEED_NDR;
1914 		}
1915 
1916 		return;
1917 	}
1918 
1919 	switch (lanes) {
1920 	case 1:
1921 		*width = IB_WIDTH_1X;
1922 		break;
1923 	case 2:
1924 		*width = IB_WIDTH_2X;
1925 		break;
1926 	case 4:
1927 		*width = IB_WIDTH_4X;
1928 		break;
1929 	case 8:
1930 		*width = IB_WIDTH_8X;
1931 		break;
1932 	case 12:
1933 		*width = IB_WIDTH_12X;
1934 		break;
1935 	default:
1936 		*width = IB_WIDTH_1X;
1937 	}
1938 
1939 	switch (netdev_speed / lanes) {
1940 	case SPEED_2500:
1941 		*speed = IB_SPEED_SDR;
1942 		break;
1943 	case SPEED_5000:
1944 		*speed = IB_SPEED_DDR;
1945 		break;
1946 	case SPEED_10000:
1947 		*speed = IB_SPEED_FDR10;
1948 		break;
1949 	case SPEED_14000:
1950 		*speed = IB_SPEED_FDR;
1951 		break;
1952 	case SPEED_25000:
1953 		*speed = IB_SPEED_EDR;
1954 		break;
1955 	case SPEED_50000:
1956 		*speed = IB_SPEED_HDR;
1957 		break;
1958 	case SPEED_100000:
1959 		*speed = IB_SPEED_NDR;
1960 		break;
1961 	default:
1962 		*speed = IB_SPEED_SDR;
1963 	}
1964 }
1965 
ib_get_eth_speed(struct ib_device * dev,u32 port_num,u16 * speed,u8 * width)1966 int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
1967 {
1968 	int rc;
1969 	u32 netdev_speed;
1970 	struct net_device *netdev;
1971 	struct ethtool_link_ksettings lksettings = {};
1972 
1973 	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1974 		return -EINVAL;
1975 
1976 	netdev = ib_device_get_netdev(dev, port_num);
1977 	if (!netdev)
1978 		return -ENODEV;
1979 
1980 	rtnl_lock();
1981 	rc = __ethtool_get_link_ksettings(netdev, &lksettings);
1982 	rtnl_unlock();
1983 
1984 	dev_put(netdev);
1985 
1986 	if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) {
1987 		netdev_speed = lksettings.base.speed;
1988 	} else {
1989 		netdev_speed = SPEED_1000;
1990 		if (rc)
1991 			pr_warn("%s speed is unknown, defaulting to %u\n",
1992 				netdev->name, netdev_speed);
1993 	}
1994 
1995 	ib_get_width_and_speed(netdev_speed, lksettings.lanes,
1996 			       speed, width);
1997 
1998 	return 0;
1999 }
2000 EXPORT_SYMBOL(ib_get_eth_speed);
2001 
ib_modify_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask)2002 int ib_modify_qp(struct ib_qp *qp,
2003 		 struct ib_qp_attr *qp_attr,
2004 		 int qp_attr_mask)
2005 {
2006 	return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
2007 }
2008 EXPORT_SYMBOL(ib_modify_qp);
2009 
ib_query_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)2010 int ib_query_qp(struct ib_qp *qp,
2011 		struct ib_qp_attr *qp_attr,
2012 		int qp_attr_mask,
2013 		struct ib_qp_init_attr *qp_init_attr)
2014 {
2015 	qp_attr->ah_attr.grh.sgid_attr = NULL;
2016 	qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
2017 
2018 	return qp->device->ops.query_qp ?
2019 		qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
2020 					 qp_init_attr) : -EOPNOTSUPP;
2021 }
2022 EXPORT_SYMBOL(ib_query_qp);
2023 
ib_close_qp(struct ib_qp * qp)2024 int ib_close_qp(struct ib_qp *qp)
2025 {
2026 	struct ib_qp *real_qp;
2027 	unsigned long flags;
2028 
2029 	real_qp = qp->real_qp;
2030 	if (real_qp == qp)
2031 		return -EINVAL;
2032 
2033 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
2034 	list_del(&qp->open_list);
2035 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
2036 
2037 	atomic_dec(&real_qp->usecnt);
2038 	if (qp->qp_sec)
2039 		ib_close_shared_qp_security(qp->qp_sec);
2040 	kfree(qp);
2041 
2042 	return 0;
2043 }
2044 EXPORT_SYMBOL(ib_close_qp);
2045 
__ib_destroy_shared_qp(struct ib_qp * qp)2046 static int __ib_destroy_shared_qp(struct ib_qp *qp)
2047 {
2048 	struct ib_xrcd *xrcd;
2049 	struct ib_qp *real_qp;
2050 	int ret;
2051 
2052 	real_qp = qp->real_qp;
2053 	xrcd = real_qp->xrcd;
2054 	down_write(&xrcd->tgt_qps_rwsem);
2055 	ib_close_qp(qp);
2056 	if (atomic_read(&real_qp->usecnt) == 0)
2057 		xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
2058 	else
2059 		real_qp = NULL;
2060 	up_write(&xrcd->tgt_qps_rwsem);
2061 
2062 	if (real_qp) {
2063 		ret = ib_destroy_qp(real_qp);
2064 		if (!ret)
2065 			atomic_dec(&xrcd->usecnt);
2066 	}
2067 
2068 	return 0;
2069 }
2070 
ib_destroy_qp_user(struct ib_qp * qp,struct ib_udata * udata)2071 int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
2072 {
2073 	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
2074 	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
2075 	struct ib_qp_security *sec;
2076 	int ret;
2077 
2078 	WARN_ON_ONCE(qp->mrs_used > 0);
2079 
2080 	if (atomic_read(&qp->usecnt))
2081 		return -EBUSY;
2082 
2083 	if (qp->real_qp != qp)
2084 		return __ib_destroy_shared_qp(qp);
2085 
2086 	sec  = qp->qp_sec;
2087 	if (sec)
2088 		ib_destroy_qp_security_begin(sec);
2089 
2090 	if (!qp->uobject)
2091 		rdma_rw_cleanup_mrs(qp);
2092 
2093 	rdma_counter_unbind_qp(qp, true);
2094 	ret = qp->device->ops.destroy_qp(qp, udata);
2095 	if (ret) {
2096 		if (sec)
2097 			ib_destroy_qp_security_abort(sec);
2098 		return ret;
2099 	}
2100 
2101 	if (alt_path_sgid_attr)
2102 		rdma_put_gid_attr(alt_path_sgid_attr);
2103 	if (av_sgid_attr)
2104 		rdma_put_gid_attr(av_sgid_attr);
2105 
2106 	ib_qp_usecnt_dec(qp);
2107 	if (sec)
2108 		ib_destroy_qp_security_end(sec);
2109 
2110 	rdma_restrack_del(&qp->res);
2111 	kfree(qp);
2112 	return ret;
2113 }
2114 EXPORT_SYMBOL(ib_destroy_qp_user);
2115 
2116 /* Completion queues */
2117 
__ib_create_cq(struct ib_device * device,ib_comp_handler comp_handler,void (* event_handler)(struct ib_event *,void *),void * cq_context,const struct ib_cq_init_attr * cq_attr,const char * caller)2118 struct ib_cq *__ib_create_cq(struct ib_device *device,
2119 			     ib_comp_handler comp_handler,
2120 			     void (*event_handler)(struct ib_event *, void *),
2121 			     void *cq_context,
2122 			     const struct ib_cq_init_attr *cq_attr,
2123 			     const char *caller)
2124 {
2125 	struct ib_cq *cq;
2126 	int ret;
2127 
2128 	cq = rdma_zalloc_drv_obj(device, ib_cq);
2129 	if (!cq)
2130 		return ERR_PTR(-ENOMEM);
2131 
2132 	cq->device = device;
2133 	cq->uobject = NULL;
2134 	cq->comp_handler = comp_handler;
2135 	cq->event_handler = event_handler;
2136 	cq->cq_context = cq_context;
2137 	atomic_set(&cq->usecnt, 0);
2138 
2139 	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
2140 	rdma_restrack_set_name(&cq->res, caller);
2141 
2142 	ret = device->ops.create_cq(cq, cq_attr, NULL);
2143 	if (ret) {
2144 		rdma_restrack_put(&cq->res);
2145 		kfree(cq);
2146 		return ERR_PTR(ret);
2147 	}
2148 
2149 	rdma_restrack_add(&cq->res);
2150 	return cq;
2151 }
2152 EXPORT_SYMBOL(__ib_create_cq);
2153 
rdma_set_cq_moderation(struct ib_cq * cq,u16 cq_count,u16 cq_period)2154 int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2155 {
2156 	if (cq->shared)
2157 		return -EOPNOTSUPP;
2158 
2159 	return cq->device->ops.modify_cq ?
2160 		cq->device->ops.modify_cq(cq, cq_count,
2161 					  cq_period) : -EOPNOTSUPP;
2162 }
2163 EXPORT_SYMBOL(rdma_set_cq_moderation);
2164 
ib_destroy_cq_user(struct ib_cq * cq,struct ib_udata * udata)2165 int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
2166 {
2167 	int ret;
2168 
2169 	if (WARN_ON_ONCE(cq->shared))
2170 		return -EOPNOTSUPP;
2171 
2172 	if (atomic_read(&cq->usecnt))
2173 		return -EBUSY;
2174 
2175 	ret = cq->device->ops.destroy_cq(cq, udata);
2176 	if (ret)
2177 		return ret;
2178 
2179 	rdma_restrack_del(&cq->res);
2180 	kfree(cq);
2181 	return ret;
2182 }
2183 EXPORT_SYMBOL(ib_destroy_cq_user);
2184 
ib_resize_cq(struct ib_cq * cq,int cqe)2185 int ib_resize_cq(struct ib_cq *cq, int cqe)
2186 {
2187 	if (cq->shared)
2188 		return -EOPNOTSUPP;
2189 
2190 	return cq->device->ops.resize_cq ?
2191 		cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
2192 }
2193 EXPORT_SYMBOL(ib_resize_cq);
2194 
2195 /* Memory regions */
2196 
ib_reg_user_mr(struct ib_pd * pd,u64 start,u64 length,u64 virt_addr,int access_flags)2197 struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2198 			     u64 virt_addr, int access_flags)
2199 {
2200 	struct ib_mr *mr;
2201 
2202 	if (access_flags & IB_ACCESS_ON_DEMAND) {
2203 		if (!(pd->device->attrs.kernel_cap_flags &
2204 		      IBK_ON_DEMAND_PAGING)) {
2205 			pr_debug("ODP support not available\n");
2206 			return ERR_PTR(-EINVAL);
2207 		}
2208 	}
2209 
2210 	mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
2211 					 access_flags, NULL);
2212 
2213 	if (IS_ERR(mr))
2214 		return mr;
2215 
2216 	mr->device = pd->device;
2217 	mr->type = IB_MR_TYPE_USER;
2218 	mr->pd = pd;
2219 	mr->dm = NULL;
2220 	atomic_inc(&pd->usecnt);
2221 	mr->iova =  virt_addr;
2222 	mr->length = length;
2223 
2224 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2225 	rdma_restrack_parent_name(&mr->res, &pd->res);
2226 	rdma_restrack_add(&mr->res);
2227 
2228 	return mr;
2229 }
2230 EXPORT_SYMBOL(ib_reg_user_mr);
2231 
ib_advise_mr(struct ib_pd * pd,enum ib_uverbs_advise_mr_advice advice,u32 flags,struct ib_sge * sg_list,u32 num_sge)2232 int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
2233 		 u32 flags, struct ib_sge *sg_list, u32 num_sge)
2234 {
2235 	if (!pd->device->ops.advise_mr)
2236 		return -EOPNOTSUPP;
2237 
2238 	if (!num_sge)
2239 		return 0;
2240 
2241 	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
2242 					 NULL);
2243 }
2244 EXPORT_SYMBOL(ib_advise_mr);
2245 
ib_dereg_mr_user(struct ib_mr * mr,struct ib_udata * udata)2246 int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
2247 {
2248 	struct ib_pd *pd = mr->pd;
2249 	struct ib_dm *dm = mr->dm;
2250 	struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
2251 	int ret;
2252 
2253 	trace_mr_dereg(mr);
2254 	rdma_restrack_del(&mr->res);
2255 	ret = mr->device->ops.dereg_mr(mr, udata);
2256 	if (!ret) {
2257 		atomic_dec(&pd->usecnt);
2258 		if (dm)
2259 			atomic_dec(&dm->usecnt);
2260 		kfree(sig_attrs);
2261 	}
2262 
2263 	return ret;
2264 }
2265 EXPORT_SYMBOL(ib_dereg_mr_user);
2266 
2267 /**
2268  * ib_alloc_mr() - Allocates a memory region
2269  * @pd:            protection domain associated with the region
2270  * @mr_type:       memory region type
2271  * @max_num_sg:    maximum sg entries available for registration.
2272  *
2273  * Notes:
2274  * Memory registeration page/sg lists must not exceed max_num_sg.
2275  * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
2276  * max_num_sg * used_page_size.
2277  *
2278  */
ib_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_num_sg)2279 struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
2280 			  u32 max_num_sg)
2281 {
2282 	struct ib_mr *mr;
2283 
2284 	if (!pd->device->ops.alloc_mr) {
2285 		mr = ERR_PTR(-EOPNOTSUPP);
2286 		goto out;
2287 	}
2288 
2289 	if (mr_type == IB_MR_TYPE_INTEGRITY) {
2290 		WARN_ON_ONCE(1);
2291 		mr = ERR_PTR(-EINVAL);
2292 		goto out;
2293 	}
2294 
2295 	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
2296 	if (IS_ERR(mr))
2297 		goto out;
2298 
2299 	mr->device = pd->device;
2300 	mr->pd = pd;
2301 	mr->dm = NULL;
2302 	mr->uobject = NULL;
2303 	atomic_inc(&pd->usecnt);
2304 	mr->need_inval = false;
2305 	mr->type = mr_type;
2306 	mr->sig_attrs = NULL;
2307 
2308 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2309 	rdma_restrack_parent_name(&mr->res, &pd->res);
2310 	rdma_restrack_add(&mr->res);
2311 out:
2312 	trace_mr_alloc(pd, mr_type, max_num_sg, mr);
2313 	return mr;
2314 }
2315 EXPORT_SYMBOL(ib_alloc_mr);
2316 
2317 /**
2318  * ib_alloc_mr_integrity() - Allocates an integrity memory region
2319  * @pd:                      protection domain associated with the region
2320  * @max_num_data_sg:         maximum data sg entries available for registration
2321  * @max_num_meta_sg:         maximum metadata sg entries available for
2322  *                           registration
2323  *
2324  * Notes:
2325  * Memory registration page/sg lists must not exceed max_num_sg,
2326  * also the integrity page/sg lists must not exceed max_num_meta_sg.
2327  *
2328  */
ib_alloc_mr_integrity(struct ib_pd * pd,u32 max_num_data_sg,u32 max_num_meta_sg)2329 struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
2330 				    u32 max_num_data_sg,
2331 				    u32 max_num_meta_sg)
2332 {
2333 	struct ib_mr *mr;
2334 	struct ib_sig_attrs *sig_attrs;
2335 
2336 	if (!pd->device->ops.alloc_mr_integrity ||
2337 	    !pd->device->ops.map_mr_sg_pi) {
2338 		mr = ERR_PTR(-EOPNOTSUPP);
2339 		goto out;
2340 	}
2341 
2342 	if (!max_num_meta_sg) {
2343 		mr = ERR_PTR(-EINVAL);
2344 		goto out;
2345 	}
2346 
2347 	sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
2348 	if (!sig_attrs) {
2349 		mr = ERR_PTR(-ENOMEM);
2350 		goto out;
2351 	}
2352 
2353 	mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
2354 						max_num_meta_sg);
2355 	if (IS_ERR(mr)) {
2356 		kfree(sig_attrs);
2357 		goto out;
2358 	}
2359 
2360 	mr->device = pd->device;
2361 	mr->pd = pd;
2362 	mr->dm = NULL;
2363 	mr->uobject = NULL;
2364 	atomic_inc(&pd->usecnt);
2365 	mr->need_inval = false;
2366 	mr->type = IB_MR_TYPE_INTEGRITY;
2367 	mr->sig_attrs = sig_attrs;
2368 
2369 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2370 	rdma_restrack_parent_name(&mr->res, &pd->res);
2371 	rdma_restrack_add(&mr->res);
2372 out:
2373 	trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
2374 	return mr;
2375 }
2376 EXPORT_SYMBOL(ib_alloc_mr_integrity);
2377 
2378 /* Multicast groups */
2379 
is_valid_mcast_lid(struct ib_qp * qp,u16 lid)2380 static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
2381 {
2382 	struct ib_qp_init_attr init_attr = {};
2383 	struct ib_qp_attr attr = {};
2384 	int num_eth_ports = 0;
2385 	unsigned int port;
2386 
2387 	/* If QP state >= init, it is assigned to a port and we can check this
2388 	 * port only.
2389 	 */
2390 	if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
2391 		if (attr.qp_state >= IB_QPS_INIT) {
2392 			if (rdma_port_get_link_layer(qp->device, attr.port_num) !=
2393 			    IB_LINK_LAYER_INFINIBAND)
2394 				return true;
2395 			goto lid_check;
2396 		}
2397 	}
2398 
2399 	/* Can't get a quick answer, iterate over all ports */
2400 	rdma_for_each_port(qp->device, port)
2401 		if (rdma_port_get_link_layer(qp->device, port) !=
2402 		    IB_LINK_LAYER_INFINIBAND)
2403 			num_eth_ports++;
2404 
2405 	/* If we have at lease one Ethernet port, RoCE annex declares that
2406 	 * multicast LID should be ignored. We can't tell at this step if the
2407 	 * QP belongs to an IB or Ethernet port.
2408 	 */
2409 	if (num_eth_ports)
2410 		return true;
2411 
2412 	/* If all the ports are IB, we can check according to IB spec. */
2413 lid_check:
2414 	return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
2415 		 lid == be16_to_cpu(IB_LID_PERMISSIVE));
2416 }
2417 
ib_attach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)2418 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2419 {
2420 	int ret;
2421 
2422 	if (!qp->device->ops.attach_mcast)
2423 		return -EOPNOTSUPP;
2424 
2425 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2426 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2427 		return -EINVAL;
2428 
2429 	ret = qp->device->ops.attach_mcast(qp, gid, lid);
2430 	if (!ret)
2431 		atomic_inc(&qp->usecnt);
2432 	return ret;
2433 }
2434 EXPORT_SYMBOL(ib_attach_mcast);
2435 
ib_detach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)2436 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2437 {
2438 	int ret;
2439 
2440 	if (!qp->device->ops.detach_mcast)
2441 		return -EOPNOTSUPP;
2442 
2443 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2444 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2445 		return -EINVAL;
2446 
2447 	ret = qp->device->ops.detach_mcast(qp, gid, lid);
2448 	if (!ret)
2449 		atomic_dec(&qp->usecnt);
2450 	return ret;
2451 }
2452 EXPORT_SYMBOL(ib_detach_mcast);
2453 
2454 /**
2455  * ib_alloc_xrcd_user - Allocates an XRC domain.
2456  * @device: The device on which to allocate the XRC domain.
2457  * @inode: inode to connect XRCD
2458  * @udata: Valid user data or NULL for kernel object
2459  */
ib_alloc_xrcd_user(struct ib_device * device,struct inode * inode,struct ib_udata * udata)2460 struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
2461 				   struct inode *inode, struct ib_udata *udata)
2462 {
2463 	struct ib_xrcd *xrcd;
2464 	int ret;
2465 
2466 	if (!device->ops.alloc_xrcd)
2467 		return ERR_PTR(-EOPNOTSUPP);
2468 
2469 	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
2470 	if (!xrcd)
2471 		return ERR_PTR(-ENOMEM);
2472 
2473 	xrcd->device = device;
2474 	xrcd->inode = inode;
2475 	atomic_set(&xrcd->usecnt, 0);
2476 	init_rwsem(&xrcd->tgt_qps_rwsem);
2477 	xa_init(&xrcd->tgt_qps);
2478 
2479 	ret = device->ops.alloc_xrcd(xrcd, udata);
2480 	if (ret)
2481 		goto err;
2482 	return xrcd;
2483 err:
2484 	kfree(xrcd);
2485 	return ERR_PTR(ret);
2486 }
2487 EXPORT_SYMBOL(ib_alloc_xrcd_user);
2488 
2489 /**
2490  * ib_dealloc_xrcd_user - Deallocates an XRC domain.
2491  * @xrcd: The XRC domain to deallocate.
2492  * @udata: Valid user data or NULL for kernel object
2493  */
ib_dealloc_xrcd_user(struct ib_xrcd * xrcd,struct ib_udata * udata)2494 int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)
2495 {
2496 	int ret;
2497 
2498 	if (atomic_read(&xrcd->usecnt))
2499 		return -EBUSY;
2500 
2501 	WARN_ON(!xa_empty(&xrcd->tgt_qps));
2502 	ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2503 	if (ret)
2504 		return ret;
2505 	kfree(xrcd);
2506 	return ret;
2507 }
2508 EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2509 
2510 /**
2511  * ib_create_wq - Creates a WQ associated with the specified protection
2512  * domain.
2513  * @pd: The protection domain associated with the WQ.
2514  * @wq_attr: A list of initial attributes required to create the
2515  * WQ. If WQ creation succeeds, then the attributes are updated to
2516  * the actual capabilities of the created WQ.
2517  *
2518  * wq_attr->max_wr and wq_attr->max_sge determine
2519  * the requested size of the WQ, and set to the actual values allocated
2520  * on return.
2521  * If ib_create_wq() succeeds, then max_wr and max_sge will always be
2522  * at least as large as the requested values.
2523  */
ib_create_wq(struct ib_pd * pd,struct ib_wq_init_attr * wq_attr)2524 struct ib_wq *ib_create_wq(struct ib_pd *pd,
2525 			   struct ib_wq_init_attr *wq_attr)
2526 {
2527 	struct ib_wq *wq;
2528 
2529 	if (!pd->device->ops.create_wq)
2530 		return ERR_PTR(-EOPNOTSUPP);
2531 
2532 	wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2533 	if (!IS_ERR(wq)) {
2534 		wq->event_handler = wq_attr->event_handler;
2535 		wq->wq_context = wq_attr->wq_context;
2536 		wq->wq_type = wq_attr->wq_type;
2537 		wq->cq = wq_attr->cq;
2538 		wq->device = pd->device;
2539 		wq->pd = pd;
2540 		wq->uobject = NULL;
2541 		atomic_inc(&pd->usecnt);
2542 		atomic_inc(&wq_attr->cq->usecnt);
2543 		atomic_set(&wq->usecnt, 0);
2544 	}
2545 	return wq;
2546 }
2547 EXPORT_SYMBOL(ib_create_wq);
2548 
2549 /**
2550  * ib_destroy_wq_user - Destroys the specified user WQ.
2551  * @wq: The WQ to destroy.
2552  * @udata: Valid user data
2553  */
ib_destroy_wq_user(struct ib_wq * wq,struct ib_udata * udata)2554 int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata)
2555 {
2556 	struct ib_cq *cq = wq->cq;
2557 	struct ib_pd *pd = wq->pd;
2558 	int ret;
2559 
2560 	if (atomic_read(&wq->usecnt))
2561 		return -EBUSY;
2562 
2563 	ret = wq->device->ops.destroy_wq(wq, udata);
2564 	if (ret)
2565 		return ret;
2566 
2567 	atomic_dec(&pd->usecnt);
2568 	atomic_dec(&cq->usecnt);
2569 	return ret;
2570 }
2571 EXPORT_SYMBOL(ib_destroy_wq_user);
2572 
ib_check_mr_status(struct ib_mr * mr,u32 check_mask,struct ib_mr_status * mr_status)2573 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2574 		       struct ib_mr_status *mr_status)
2575 {
2576 	if (!mr->device->ops.check_mr_status)
2577 		return -EOPNOTSUPP;
2578 
2579 	return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2580 }
2581 EXPORT_SYMBOL(ib_check_mr_status);
2582 
ib_set_vf_link_state(struct ib_device * device,int vf,u32 port,int state)2583 int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
2584 			 int state)
2585 {
2586 	if (!device->ops.set_vf_link_state)
2587 		return -EOPNOTSUPP;
2588 
2589 	return device->ops.set_vf_link_state(device, vf, port, state);
2590 }
2591 EXPORT_SYMBOL(ib_set_vf_link_state);
2592 
ib_get_vf_config(struct ib_device * device,int vf,u32 port,struct ifla_vf_info * info)2593 int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
2594 		     struct ifla_vf_info *info)
2595 {
2596 	if (!device->ops.get_vf_config)
2597 		return -EOPNOTSUPP;
2598 
2599 	return device->ops.get_vf_config(device, vf, port, info);
2600 }
2601 EXPORT_SYMBOL(ib_get_vf_config);
2602 
ib_get_vf_stats(struct ib_device * device,int vf,u32 port,struct ifla_vf_stats * stats)2603 int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
2604 		    struct ifla_vf_stats *stats)
2605 {
2606 	if (!device->ops.get_vf_stats)
2607 		return -EOPNOTSUPP;
2608 
2609 	return device->ops.get_vf_stats(device, vf, port, stats);
2610 }
2611 EXPORT_SYMBOL(ib_get_vf_stats);
2612 
ib_set_vf_guid(struct ib_device * device,int vf,u32 port,u64 guid,int type)2613 int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
2614 		   int type)
2615 {
2616 	if (!device->ops.set_vf_guid)
2617 		return -EOPNOTSUPP;
2618 
2619 	return device->ops.set_vf_guid(device, vf, port, guid, type);
2620 }
2621 EXPORT_SYMBOL(ib_set_vf_guid);
2622 
ib_get_vf_guid(struct ib_device * device,int vf,u32 port,struct ifla_vf_guid * node_guid,struct ifla_vf_guid * port_guid)2623 int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
2624 		   struct ifla_vf_guid *node_guid,
2625 		   struct ifla_vf_guid *port_guid)
2626 {
2627 	if (!device->ops.get_vf_guid)
2628 		return -EOPNOTSUPP;
2629 
2630 	return device->ops.get_vf_guid(device, vf, port, node_guid, port_guid);
2631 }
2632 EXPORT_SYMBOL(ib_get_vf_guid);
2633 /**
2634  * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
2635  *     information) and set an appropriate memory region for registration.
2636  * @mr:             memory region
2637  * @data_sg:        dma mapped scatterlist for data
2638  * @data_sg_nents:  number of entries in data_sg
2639  * @data_sg_offset: offset in bytes into data_sg
2640  * @meta_sg:        dma mapped scatterlist for metadata
2641  * @meta_sg_nents:  number of entries in meta_sg
2642  * @meta_sg_offset: offset in bytes into meta_sg
2643  * @page_size:      page vector desired page size
2644  *
2645  * Constraints:
2646  * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
2647  *
2648  * Return: 0 on success.
2649  *
2650  * After this completes successfully, the  memory region
2651  * is ready for registration.
2652  */
ib_map_mr_sg_pi(struct ib_mr * mr,struct scatterlist * data_sg,int data_sg_nents,unsigned int * data_sg_offset,struct scatterlist * meta_sg,int meta_sg_nents,unsigned int * meta_sg_offset,unsigned int page_size)2653 int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
2654 		    int data_sg_nents, unsigned int *data_sg_offset,
2655 		    struct scatterlist *meta_sg, int meta_sg_nents,
2656 		    unsigned int *meta_sg_offset, unsigned int page_size)
2657 {
2658 	if (unlikely(!mr->device->ops.map_mr_sg_pi ||
2659 		     WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
2660 		return -EOPNOTSUPP;
2661 
2662 	mr->page_size = page_size;
2663 
2664 	return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
2665 					    data_sg_offset, meta_sg,
2666 					    meta_sg_nents, meta_sg_offset);
2667 }
2668 EXPORT_SYMBOL(ib_map_mr_sg_pi);
2669 
2670 /**
2671  * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
2672  *     and set it the memory region.
2673  * @mr:            memory region
2674  * @sg:            dma mapped scatterlist
2675  * @sg_nents:      number of entries in sg
2676  * @sg_offset:     offset in bytes into sg
2677  * @page_size:     page vector desired page size
2678  *
2679  * Constraints:
2680  *
2681  * - The first sg element is allowed to have an offset.
2682  * - Each sg element must either be aligned to page_size or virtually
2683  *   contiguous to the previous element. In case an sg element has a
2684  *   non-contiguous offset, the mapping prefix will not include it.
2685  * - The last sg element is allowed to have length less than page_size.
2686  * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
2687  *   then only max_num_sg entries will be mapped.
2688  * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS, none of these
2689  *   constraints holds and the page_size argument is ignored.
2690  *
2691  * Returns the number of sg elements that were mapped to the memory region.
2692  *
2693  * After this completes successfully, the  memory region
2694  * is ready for registration.
2695  */
ib_map_mr_sg(struct ib_mr * mr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset,unsigned int page_size)2696 int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2697 		 unsigned int *sg_offset, unsigned int page_size)
2698 {
2699 	if (unlikely(!mr->device->ops.map_mr_sg))
2700 		return -EOPNOTSUPP;
2701 
2702 	mr->page_size = page_size;
2703 
2704 	return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2705 }
2706 EXPORT_SYMBOL(ib_map_mr_sg);
2707 
2708 /**
2709  * ib_sg_to_pages() - Convert the largest prefix of a sg list
2710  *     to a page vector
2711  * @mr:            memory region
2712  * @sgl:           dma mapped scatterlist
2713  * @sg_nents:      number of entries in sg
2714  * @sg_offset_p:   ==== =======================================================
2715  *                 IN   start offset in bytes into sg
2716  *                 OUT  offset in bytes for element n of the sg of the first
2717  *                      byte that has not been processed where n is the return
2718  *                      value of this function.
2719  *                 ==== =======================================================
2720  * @set_page:      driver page assignment function pointer
2721  *
2722  * Core service helper for drivers to convert the largest
2723  * prefix of given sg list to a page vector. The sg list
2724  * prefix converted is the prefix that meet the requirements
2725  * of ib_map_mr_sg.
2726  *
2727  * Returns the number of sg elements that were assigned to
2728  * a page vector.
2729  */
ib_sg_to_pages(struct ib_mr * mr,struct scatterlist * sgl,int sg_nents,unsigned int * sg_offset_p,int (* set_page)(struct ib_mr *,u64))2730 int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
2731 		unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
2732 {
2733 	struct scatterlist *sg;
2734 	u64 last_end_dma_addr = 0;
2735 	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2736 	unsigned int last_page_off = 0;
2737 	u64 page_mask = ~((u64)mr->page_size - 1);
2738 	int i, ret;
2739 
2740 	if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
2741 		return -EINVAL;
2742 
2743 	mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
2744 	mr->length = 0;
2745 
2746 	for_each_sg(sgl, sg, sg_nents, i) {
2747 		u64 dma_addr = sg_dma_address(sg) + sg_offset;
2748 		u64 prev_addr = dma_addr;
2749 		unsigned int dma_len = sg_dma_len(sg) - sg_offset;
2750 		u64 end_dma_addr = dma_addr + dma_len;
2751 		u64 page_addr = dma_addr & page_mask;
2752 
2753 		/*
2754 		 * For the second and later elements, check whether either the
2755 		 * end of element i-1 or the start of element i is not aligned
2756 		 * on a page boundary.
2757 		 */
2758 		if (i && (last_page_off != 0 || page_addr != dma_addr)) {
2759 			/* Stop mapping if there is a gap. */
2760 			if (last_end_dma_addr != dma_addr)
2761 				break;
2762 
2763 			/*
2764 			 * Coalesce this element with the last. If it is small
2765 			 * enough just update mr->length. Otherwise start
2766 			 * mapping from the next page.
2767 			 */
2768 			goto next_page;
2769 		}
2770 
2771 		do {
2772 			ret = set_page(mr, page_addr);
2773 			if (unlikely(ret < 0)) {
2774 				sg_offset = prev_addr - sg_dma_address(sg);
2775 				mr->length += prev_addr - dma_addr;
2776 				if (sg_offset_p)
2777 					*sg_offset_p = sg_offset;
2778 				return i || sg_offset ? i : ret;
2779 			}
2780 			prev_addr = page_addr;
2781 next_page:
2782 			page_addr += mr->page_size;
2783 		} while (page_addr < end_dma_addr);
2784 
2785 		mr->length += dma_len;
2786 		last_end_dma_addr = end_dma_addr;
2787 		last_page_off = end_dma_addr & ~page_mask;
2788 
2789 		sg_offset = 0;
2790 	}
2791 
2792 	if (sg_offset_p)
2793 		*sg_offset_p = 0;
2794 	return i;
2795 }
2796 EXPORT_SYMBOL(ib_sg_to_pages);
2797 
2798 struct ib_drain_cqe {
2799 	struct ib_cqe cqe;
2800 	struct completion done;
2801 };
2802 
ib_drain_qp_done(struct ib_cq * cq,struct ib_wc * wc)2803 static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
2804 {
2805 	struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
2806 						cqe);
2807 
2808 	complete(&cqe->done);
2809 }
2810 
2811 /*
2812  * Post a WR and block until its completion is reaped for the SQ.
2813  */
__ib_drain_sq(struct ib_qp * qp)2814 static void __ib_drain_sq(struct ib_qp *qp)
2815 {
2816 	struct ib_cq *cq = qp->send_cq;
2817 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2818 	struct ib_drain_cqe sdrain;
2819 	struct ib_rdma_wr swr = {
2820 		.wr = {
2821 			.next = NULL,
2822 			{ .wr_cqe	= &sdrain.cqe, },
2823 			.opcode	= IB_WR_RDMA_WRITE,
2824 		},
2825 	};
2826 	int ret;
2827 
2828 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2829 	if (ret) {
2830 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2831 		return;
2832 	}
2833 
2834 	sdrain.cqe.done = ib_drain_qp_done;
2835 	init_completion(&sdrain.done);
2836 
2837 	ret = ib_post_send(qp, &swr.wr, NULL);
2838 	if (ret) {
2839 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2840 		return;
2841 	}
2842 
2843 	if (cq->poll_ctx == IB_POLL_DIRECT)
2844 		while (wait_for_completion_timeout(&sdrain.done, HZ / 10) <= 0)
2845 			ib_process_cq_direct(cq, -1);
2846 	else
2847 		wait_for_completion(&sdrain.done);
2848 }
2849 
2850 /*
2851  * Post a WR and block until its completion is reaped for the RQ.
2852  */
__ib_drain_rq(struct ib_qp * qp)2853 static void __ib_drain_rq(struct ib_qp *qp)
2854 {
2855 	struct ib_cq *cq = qp->recv_cq;
2856 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2857 	struct ib_drain_cqe rdrain;
2858 	struct ib_recv_wr rwr = {};
2859 	int ret;
2860 
2861 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2862 	if (ret) {
2863 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2864 		return;
2865 	}
2866 
2867 	rwr.wr_cqe = &rdrain.cqe;
2868 	rdrain.cqe.done = ib_drain_qp_done;
2869 	init_completion(&rdrain.done);
2870 
2871 	ret = ib_post_recv(qp, &rwr, NULL);
2872 	if (ret) {
2873 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2874 		return;
2875 	}
2876 
2877 	if (cq->poll_ctx == IB_POLL_DIRECT)
2878 		while (wait_for_completion_timeout(&rdrain.done, HZ / 10) <= 0)
2879 			ib_process_cq_direct(cq, -1);
2880 	else
2881 		wait_for_completion(&rdrain.done);
2882 }
2883 
2884 /**
2885  * ib_drain_sq() - Block until all SQ CQEs have been consumed by the
2886  *		   application.
2887  * @qp:            queue pair to drain
2888  *
2889  * If the device has a provider-specific drain function, then
2890  * call that.  Otherwise call the generic drain function
2891  * __ib_drain_sq().
2892  *
2893  * The caller must:
2894  *
2895  * ensure there is room in the CQ and SQ for the drain work request and
2896  * completion.
2897  *
2898  * allocate the CQ using ib_alloc_cq().
2899  *
2900  * ensure that there are no other contexts that are posting WRs concurrently.
2901  * Otherwise the drain is not guaranteed.
2902  */
ib_drain_sq(struct ib_qp * qp)2903 void ib_drain_sq(struct ib_qp *qp)
2904 {
2905 	if (qp->device->ops.drain_sq)
2906 		qp->device->ops.drain_sq(qp);
2907 	else
2908 		__ib_drain_sq(qp);
2909 	trace_cq_drain_complete(qp->send_cq);
2910 }
2911 EXPORT_SYMBOL(ib_drain_sq);
2912 
2913 /**
2914  * ib_drain_rq() - Block until all RQ CQEs have been consumed by the
2915  *		   application.
2916  * @qp:            queue pair to drain
2917  *
2918  * If the device has a provider-specific drain function, then
2919  * call that.  Otherwise call the generic drain function
2920  * __ib_drain_rq().
2921  *
2922  * The caller must:
2923  *
2924  * ensure there is room in the CQ and RQ for the drain work request and
2925  * completion.
2926  *
2927  * allocate the CQ using ib_alloc_cq().
2928  *
2929  * ensure that there are no other contexts that are posting WRs concurrently.
2930  * Otherwise the drain is not guaranteed.
2931  */
ib_drain_rq(struct ib_qp * qp)2932 void ib_drain_rq(struct ib_qp *qp)
2933 {
2934 	if (qp->device->ops.drain_rq)
2935 		qp->device->ops.drain_rq(qp);
2936 	else
2937 		__ib_drain_rq(qp);
2938 	trace_cq_drain_complete(qp->recv_cq);
2939 }
2940 EXPORT_SYMBOL(ib_drain_rq);
2941 
2942 /**
2943  * ib_drain_qp() - Block until all CQEs have been consumed by the
2944  *		   application on both the RQ and SQ.
2945  * @qp:            queue pair to drain
2946  *
2947  * The caller must:
2948  *
2949  * ensure there is room in the CQ(s), SQ, and RQ for drain work requests
2950  * and completions.
2951  *
2952  * allocate the CQs using ib_alloc_cq().
2953  *
2954  * ensure that there are no other contexts that are posting WRs concurrently.
2955  * Otherwise the drain is not guaranteed.
2956  */
ib_drain_qp(struct ib_qp * qp)2957 void ib_drain_qp(struct ib_qp *qp)
2958 {
2959 	ib_drain_sq(qp);
2960 	if (!qp->srq)
2961 		ib_drain_rq(qp);
2962 }
2963 EXPORT_SYMBOL(ib_drain_qp);
2964 
rdma_alloc_netdev(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *))2965 struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
2966 				     enum rdma_netdev_t type, const char *name,
2967 				     unsigned char name_assign_type,
2968 				     void (*setup)(struct net_device *))
2969 {
2970 	struct rdma_netdev_alloc_params params;
2971 	struct net_device *netdev;
2972 	int rc;
2973 
2974 	if (!device->ops.rdma_netdev_get_params)
2975 		return ERR_PTR(-EOPNOTSUPP);
2976 
2977 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2978 						&params);
2979 	if (rc)
2980 		return ERR_PTR(rc);
2981 
2982 	netdev = alloc_netdev_mqs(params.sizeof_priv, name, name_assign_type,
2983 				  setup, params.txqs, params.rxqs);
2984 	if (!netdev)
2985 		return ERR_PTR(-ENOMEM);
2986 
2987 	return netdev;
2988 }
2989 EXPORT_SYMBOL(rdma_alloc_netdev);
2990 
rdma_init_netdev(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *),struct net_device * netdev)2991 int rdma_init_netdev(struct ib_device *device, u32 port_num,
2992 		     enum rdma_netdev_t type, const char *name,
2993 		     unsigned char name_assign_type,
2994 		     void (*setup)(struct net_device *),
2995 		     struct net_device *netdev)
2996 {
2997 	struct rdma_netdev_alloc_params params;
2998 	int rc;
2999 
3000 	if (!device->ops.rdma_netdev_get_params)
3001 		return -EOPNOTSUPP;
3002 
3003 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
3004 						&params);
3005 	if (rc)
3006 		return rc;
3007 
3008 	return params.initialize_rdma_netdev(device, port_num,
3009 					     netdev, params.param);
3010 }
3011 EXPORT_SYMBOL(rdma_init_netdev);
3012 
__rdma_block_iter_start(struct ib_block_iter * biter,struct scatterlist * sglist,unsigned int nents,unsigned long pgsz)3013 void __rdma_block_iter_start(struct ib_block_iter *biter,
3014 			     struct scatterlist *sglist, unsigned int nents,
3015 			     unsigned long pgsz)
3016 {
3017 	memset(biter, 0, sizeof(struct ib_block_iter));
3018 	biter->__sg = sglist;
3019 	biter->__sg_nents = nents;
3020 
3021 	/* Driver provides best block size to use */
3022 	biter->__pg_bit = __fls(pgsz);
3023 }
3024 EXPORT_SYMBOL(__rdma_block_iter_start);
3025 
__rdma_block_iter_next(struct ib_block_iter * biter)3026 bool __rdma_block_iter_next(struct ib_block_iter *biter)
3027 {
3028 	unsigned int block_offset;
3029 	unsigned int sg_delta;
3030 
3031 	if (!biter->__sg_nents || !biter->__sg)
3032 		return false;
3033 
3034 	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
3035 	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
3036 	sg_delta = BIT_ULL(biter->__pg_bit) - block_offset;
3037 
3038 	if (sg_dma_len(biter->__sg) - biter->__sg_advance > sg_delta) {
3039 		biter->__sg_advance += sg_delta;
3040 	} else {
3041 		biter->__sg_advance = 0;
3042 		biter->__sg = sg_next(biter->__sg);
3043 		biter->__sg_nents--;
3044 	}
3045 
3046 	return true;
3047 }
3048 EXPORT_SYMBOL(__rdma_block_iter_next);
3049 
3050 /**
3051  * rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct
3052  *   for the drivers.
3053  * @descs: array of static descriptors
3054  * @num_counters: number of elements in array
3055  * @lifespan: milliseconds between updates
3056  */
rdma_alloc_hw_stats_struct(const struct rdma_stat_desc * descs,int num_counters,unsigned long lifespan)3057 struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
3058 	const struct rdma_stat_desc *descs, int num_counters,
3059 	unsigned long lifespan)
3060 {
3061 	struct rdma_hw_stats *stats;
3062 
3063 	stats = kzalloc(struct_size(stats, value, num_counters), GFP_KERNEL);
3064 	if (!stats)
3065 		return NULL;
3066 
3067 	stats->is_disabled = kcalloc(BITS_TO_LONGS(num_counters),
3068 				     sizeof(*stats->is_disabled), GFP_KERNEL);
3069 	if (!stats->is_disabled)
3070 		goto err;
3071 
3072 	stats->descs = descs;
3073 	stats->num_counters = num_counters;
3074 	stats->lifespan = msecs_to_jiffies(lifespan);
3075 	mutex_init(&stats->lock);
3076 
3077 	return stats;
3078 
3079 err:
3080 	kfree(stats);
3081 	return NULL;
3082 }
3083 EXPORT_SYMBOL(rdma_alloc_hw_stats_struct);
3084 
3085 /**
3086  * rdma_free_hw_stats_struct - Helper function to release rdma_hw_stats
3087  * @stats: statistics to release
3088  */
rdma_free_hw_stats_struct(struct rdma_hw_stats * stats)3089 void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats)
3090 {
3091 	if (!stats)
3092 		return;
3093 
3094 	kfree(stats->is_disabled);
3095 	kfree(stats);
3096 }
3097 EXPORT_SYMBOL(rdma_free_hw_stats_struct);
3098