1 /*
2  * Copyright (c) 2004 Mellanox Technologies Ltd.  All rights reserved.
3  * Copyright (c) 2004 Infinicon Corporation.  All rights reserved.
4  * Copyright (c) 2004 Intel Corporation.  All rights reserved.
5  * Copyright (c) 2004 Topspin Corporation.  All rights reserved.
6  * Copyright (c) 2004 Voltaire Corporation.  All rights reserved.
7  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
8  * Copyright (c) 2005, 2006 Cisco Systems.  All rights reserved.
9  *
10  * This software is available to you under a choice of one of two
11  * licenses.  You may choose to be licensed under the terms of the GNU
12  * General Public License (GPL) Version 2, available from the file
13  * COPYING in the main directory of this source tree, or the
14  * OpenIB.org BSD license below:
15  *
16  *     Redistribution and use in source and binary forms, with or
17  *     without modification, are permitted provided that the following
18  *     conditions are met:
19  *
20  *      - Redistributions of source code must retain the above
21  *        copyright notice, this list of conditions and the following
22  *        disclaimer.
23  *
24  *      - Redistributions in binary form must reproduce the above
25  *        copyright notice, this list of conditions and the following
26  *        disclaimer in the documentation and/or other materials
27  *        provided with the distribution.
28  *
29  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
30  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
31  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
32  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
33  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
34  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
35  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36  * SOFTWARE.
37  */
38 
39 #include <linux/errno.h>
40 #include <linux/err.h>
41 #include <linux/export.h>
42 #include <linux/string.h>
43 #include <linux/slab.h>
44 #include <linux/in.h>
45 #include <linux/in6.h>
46 #include <net/addrconf.h>
47 #include <linux/security.h>
48 
49 #include <rdma/ib_verbs.h>
50 #include <rdma/ib_cache.h>
51 #include <rdma/ib_addr.h>
52 #include <rdma/rw.h>
53 #include <rdma/lag.h>
54 
55 #include "core_priv.h"
56 #include <trace/events/rdma_core.h>
57 
58 static int ib_resolve_eth_dmac(struct ib_device *device,
59 			       struct rdma_ah_attr *ah_attr);
60 
61 static const char * const ib_events[] = {
62 	[IB_EVENT_CQ_ERR]		= "CQ error",
63 	[IB_EVENT_QP_FATAL]		= "QP fatal error",
64 	[IB_EVENT_QP_REQ_ERR]		= "QP request error",
65 	[IB_EVENT_QP_ACCESS_ERR]	= "QP access error",
66 	[IB_EVENT_COMM_EST]		= "communication established",
67 	[IB_EVENT_SQ_DRAINED]		= "send queue drained",
68 	[IB_EVENT_PATH_MIG]		= "path migration successful",
69 	[IB_EVENT_PATH_MIG_ERR]		= "path migration error",
70 	[IB_EVENT_DEVICE_FATAL]		= "device fatal error",
71 	[IB_EVENT_PORT_ACTIVE]		= "port active",
72 	[IB_EVENT_PORT_ERR]		= "port error",
73 	[IB_EVENT_LID_CHANGE]		= "LID change",
74 	[IB_EVENT_PKEY_CHANGE]		= "P_key change",
75 	[IB_EVENT_SM_CHANGE]		= "SM change",
76 	[IB_EVENT_SRQ_ERR]		= "SRQ error",
77 	[IB_EVENT_SRQ_LIMIT_REACHED]	= "SRQ limit reached",
78 	[IB_EVENT_QP_LAST_WQE_REACHED]	= "last WQE reached",
79 	[IB_EVENT_CLIENT_REREGISTER]	= "client reregister",
80 	[IB_EVENT_GID_CHANGE]		= "GID changed",
81 };
82 
ib_event_msg(enum ib_event_type event)83 const char *__attribute_const__ ib_event_msg(enum ib_event_type event)
84 {
85 	size_t index = event;
86 
87 	return (index < ARRAY_SIZE(ib_events) && ib_events[index]) ?
88 			ib_events[index] : "unrecognized event";
89 }
90 EXPORT_SYMBOL(ib_event_msg);
91 
92 static const char * const wc_statuses[] = {
93 	[IB_WC_SUCCESS]			= "success",
94 	[IB_WC_LOC_LEN_ERR]		= "local length error",
95 	[IB_WC_LOC_QP_OP_ERR]		= "local QP operation error",
96 	[IB_WC_LOC_EEC_OP_ERR]		= "local EE context operation error",
97 	[IB_WC_LOC_PROT_ERR]		= "local protection error",
98 	[IB_WC_WR_FLUSH_ERR]		= "WR flushed",
99 	[IB_WC_MW_BIND_ERR]		= "memory bind operation error",
100 	[IB_WC_BAD_RESP_ERR]		= "bad response error",
101 	[IB_WC_LOC_ACCESS_ERR]		= "local access error",
102 	[IB_WC_REM_INV_REQ_ERR]		= "remote invalid request error",
103 	[IB_WC_REM_ACCESS_ERR]		= "remote access error",
104 	[IB_WC_REM_OP_ERR]		= "remote operation error",
105 	[IB_WC_RETRY_EXC_ERR]		= "transport retry counter exceeded",
106 	[IB_WC_RNR_RETRY_EXC_ERR]	= "RNR retry counter exceeded",
107 	[IB_WC_LOC_RDD_VIOL_ERR]	= "local RDD violation error",
108 	[IB_WC_REM_INV_RD_REQ_ERR]	= "remote invalid RD request",
109 	[IB_WC_REM_ABORT_ERR]		= "operation aborted",
110 	[IB_WC_INV_EECN_ERR]		= "invalid EE context number",
111 	[IB_WC_INV_EEC_STATE_ERR]	= "invalid EE context state",
112 	[IB_WC_FATAL_ERR]		= "fatal error",
113 	[IB_WC_RESP_TIMEOUT_ERR]	= "response timeout error",
114 	[IB_WC_GENERAL_ERR]		= "general error",
115 };
116 
ib_wc_status_msg(enum ib_wc_status status)117 const char *__attribute_const__ ib_wc_status_msg(enum ib_wc_status status)
118 {
119 	size_t index = status;
120 
121 	return (index < ARRAY_SIZE(wc_statuses) && wc_statuses[index]) ?
122 			wc_statuses[index] : "unrecognized status";
123 }
124 EXPORT_SYMBOL(ib_wc_status_msg);
125 
ib_rate_to_mult(enum ib_rate rate)126 __attribute_const__ int ib_rate_to_mult(enum ib_rate rate)
127 {
128 	switch (rate) {
129 	case IB_RATE_2_5_GBPS: return   1;
130 	case IB_RATE_5_GBPS:   return   2;
131 	case IB_RATE_10_GBPS:  return   4;
132 	case IB_RATE_20_GBPS:  return   8;
133 	case IB_RATE_30_GBPS:  return  12;
134 	case IB_RATE_40_GBPS:  return  16;
135 	case IB_RATE_60_GBPS:  return  24;
136 	case IB_RATE_80_GBPS:  return  32;
137 	case IB_RATE_120_GBPS: return  48;
138 	case IB_RATE_14_GBPS:  return   6;
139 	case IB_RATE_56_GBPS:  return  22;
140 	case IB_RATE_112_GBPS: return  45;
141 	case IB_RATE_168_GBPS: return  67;
142 	case IB_RATE_25_GBPS:  return  10;
143 	case IB_RATE_100_GBPS: return  40;
144 	case IB_RATE_200_GBPS: return  80;
145 	case IB_RATE_300_GBPS: return 120;
146 	case IB_RATE_28_GBPS:  return  11;
147 	case IB_RATE_50_GBPS:  return  20;
148 	case IB_RATE_400_GBPS: return 160;
149 	case IB_RATE_600_GBPS: return 240;
150 	default:	       return  -1;
151 	}
152 }
153 EXPORT_SYMBOL(ib_rate_to_mult);
154 
mult_to_ib_rate(int mult)155 __attribute_const__ enum ib_rate mult_to_ib_rate(int mult)
156 {
157 	switch (mult) {
158 	case 1:   return IB_RATE_2_5_GBPS;
159 	case 2:   return IB_RATE_5_GBPS;
160 	case 4:   return IB_RATE_10_GBPS;
161 	case 8:   return IB_RATE_20_GBPS;
162 	case 12:  return IB_RATE_30_GBPS;
163 	case 16:  return IB_RATE_40_GBPS;
164 	case 24:  return IB_RATE_60_GBPS;
165 	case 32:  return IB_RATE_80_GBPS;
166 	case 48:  return IB_RATE_120_GBPS;
167 	case 6:   return IB_RATE_14_GBPS;
168 	case 22:  return IB_RATE_56_GBPS;
169 	case 45:  return IB_RATE_112_GBPS;
170 	case 67:  return IB_RATE_168_GBPS;
171 	case 10:  return IB_RATE_25_GBPS;
172 	case 40:  return IB_RATE_100_GBPS;
173 	case 80:  return IB_RATE_200_GBPS;
174 	case 120: return IB_RATE_300_GBPS;
175 	case 11:  return IB_RATE_28_GBPS;
176 	case 20:  return IB_RATE_50_GBPS;
177 	case 160: return IB_RATE_400_GBPS;
178 	case 240: return IB_RATE_600_GBPS;
179 	default:  return IB_RATE_PORT_CURRENT;
180 	}
181 }
182 EXPORT_SYMBOL(mult_to_ib_rate);
183 
ib_rate_to_mbps(enum ib_rate rate)184 __attribute_const__ int ib_rate_to_mbps(enum ib_rate rate)
185 {
186 	switch (rate) {
187 	case IB_RATE_2_5_GBPS: return 2500;
188 	case IB_RATE_5_GBPS:   return 5000;
189 	case IB_RATE_10_GBPS:  return 10000;
190 	case IB_RATE_20_GBPS:  return 20000;
191 	case IB_RATE_30_GBPS:  return 30000;
192 	case IB_RATE_40_GBPS:  return 40000;
193 	case IB_RATE_60_GBPS:  return 60000;
194 	case IB_RATE_80_GBPS:  return 80000;
195 	case IB_RATE_120_GBPS: return 120000;
196 	case IB_RATE_14_GBPS:  return 14062;
197 	case IB_RATE_56_GBPS:  return 56250;
198 	case IB_RATE_112_GBPS: return 112500;
199 	case IB_RATE_168_GBPS: return 168750;
200 	case IB_RATE_25_GBPS:  return 25781;
201 	case IB_RATE_100_GBPS: return 103125;
202 	case IB_RATE_200_GBPS: return 206250;
203 	case IB_RATE_300_GBPS: return 309375;
204 	case IB_RATE_28_GBPS:  return 28125;
205 	case IB_RATE_50_GBPS:  return 53125;
206 	case IB_RATE_400_GBPS: return 425000;
207 	case IB_RATE_600_GBPS: return 637500;
208 	default:	       return -1;
209 	}
210 }
211 EXPORT_SYMBOL(ib_rate_to_mbps);
212 
213 __attribute_const__ enum rdma_transport_type
rdma_node_get_transport(unsigned int node_type)214 rdma_node_get_transport(unsigned int node_type)
215 {
216 
217 	if (node_type == RDMA_NODE_USNIC)
218 		return RDMA_TRANSPORT_USNIC;
219 	if (node_type == RDMA_NODE_USNIC_UDP)
220 		return RDMA_TRANSPORT_USNIC_UDP;
221 	if (node_type == RDMA_NODE_RNIC)
222 		return RDMA_TRANSPORT_IWARP;
223 	if (node_type == RDMA_NODE_UNSPECIFIED)
224 		return RDMA_TRANSPORT_UNSPECIFIED;
225 
226 	return RDMA_TRANSPORT_IB;
227 }
228 EXPORT_SYMBOL(rdma_node_get_transport);
229 
rdma_port_get_link_layer(struct ib_device * device,u32 port_num)230 enum rdma_link_layer rdma_port_get_link_layer(struct ib_device *device,
231 					      u32 port_num)
232 {
233 	enum rdma_transport_type lt;
234 	if (device->ops.get_link_layer)
235 		return device->ops.get_link_layer(device, port_num);
236 
237 	lt = rdma_node_get_transport(device->node_type);
238 	if (lt == RDMA_TRANSPORT_IB)
239 		return IB_LINK_LAYER_INFINIBAND;
240 
241 	return IB_LINK_LAYER_ETHERNET;
242 }
243 EXPORT_SYMBOL(rdma_port_get_link_layer);
244 
245 /* Protection domains */
246 
247 /**
248  * __ib_alloc_pd - Allocates an unused protection domain.
249  * @device: The device on which to allocate the protection domain.
250  * @flags: protection domain flags
251  * @caller: caller's build-time module name
252  *
253  * A protection domain object provides an association between QPs, shared
254  * receive queues, address handles, memory regions, and memory windows.
255  *
256  * Every PD has a local_dma_lkey which can be used as the lkey value for local
257  * memory operations.
258  */
__ib_alloc_pd(struct ib_device * device,unsigned int flags,const char * caller)259 struct ib_pd *__ib_alloc_pd(struct ib_device *device, unsigned int flags,
260 		const char *caller)
261 {
262 	struct ib_pd *pd;
263 	int mr_access_flags = 0;
264 	int ret;
265 
266 	pd = rdma_zalloc_drv_obj(device, ib_pd);
267 	if (!pd)
268 		return ERR_PTR(-ENOMEM);
269 
270 	pd->device = device;
271 	pd->flags = flags;
272 
273 	rdma_restrack_new(&pd->res, RDMA_RESTRACK_PD);
274 	rdma_restrack_set_name(&pd->res, caller);
275 
276 	ret = device->ops.alloc_pd(pd, NULL);
277 	if (ret) {
278 		rdma_restrack_put(&pd->res);
279 		kfree(pd);
280 		return ERR_PTR(ret);
281 	}
282 	rdma_restrack_add(&pd->res);
283 
284 	if (device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY)
285 		pd->local_dma_lkey = device->local_dma_lkey;
286 	else
287 		mr_access_flags |= IB_ACCESS_LOCAL_WRITE;
288 
289 	if (flags & IB_PD_UNSAFE_GLOBAL_RKEY) {
290 		pr_warn("%s: enabling unsafe global rkey\n", caller);
291 		mr_access_flags |= IB_ACCESS_REMOTE_READ | IB_ACCESS_REMOTE_WRITE;
292 	}
293 
294 	if (mr_access_flags) {
295 		struct ib_mr *mr;
296 
297 		mr = pd->device->ops.get_dma_mr(pd, mr_access_flags);
298 		if (IS_ERR(mr)) {
299 			ib_dealloc_pd(pd);
300 			return ERR_CAST(mr);
301 		}
302 
303 		mr->device	= pd->device;
304 		mr->pd		= pd;
305 		mr->type        = IB_MR_TYPE_DMA;
306 		mr->uobject	= NULL;
307 		mr->need_inval	= false;
308 
309 		pd->__internal_mr = mr;
310 
311 		if (!(device->attrs.kernel_cap_flags & IBK_LOCAL_DMA_LKEY))
312 			pd->local_dma_lkey = pd->__internal_mr->lkey;
313 
314 		if (flags & IB_PD_UNSAFE_GLOBAL_RKEY)
315 			pd->unsafe_global_rkey = pd->__internal_mr->rkey;
316 	}
317 
318 	return pd;
319 }
320 EXPORT_SYMBOL(__ib_alloc_pd);
321 
322 /**
323  * ib_dealloc_pd_user - Deallocates a protection domain.
324  * @pd: The protection domain to deallocate.
325  * @udata: Valid user data or NULL for kernel object
326  *
327  * It is an error to call this function while any resources in the pd still
328  * exist.  The caller is responsible to synchronously destroy them and
329  * guarantee no new allocations will happen.
330  */
ib_dealloc_pd_user(struct ib_pd * pd,struct ib_udata * udata)331 int ib_dealloc_pd_user(struct ib_pd *pd, struct ib_udata *udata)
332 {
333 	int ret;
334 
335 	if (pd->__internal_mr) {
336 		ret = pd->device->ops.dereg_mr(pd->__internal_mr, NULL);
337 		WARN_ON(ret);
338 		pd->__internal_mr = NULL;
339 	}
340 
341 	ret = pd->device->ops.dealloc_pd(pd, udata);
342 	if (ret)
343 		return ret;
344 
345 	rdma_restrack_del(&pd->res);
346 	kfree(pd);
347 	return ret;
348 }
349 EXPORT_SYMBOL(ib_dealloc_pd_user);
350 
351 /* Address handles */
352 
353 /**
354  * rdma_copy_ah_attr - Copy rdma ah attribute from source to destination.
355  * @dest:       Pointer to destination ah_attr. Contents of the destination
356  *              pointer is assumed to be invalid and attribute are overwritten.
357  * @src:        Pointer to source ah_attr.
358  */
rdma_copy_ah_attr(struct rdma_ah_attr * dest,const struct rdma_ah_attr * src)359 void rdma_copy_ah_attr(struct rdma_ah_attr *dest,
360 		       const struct rdma_ah_attr *src)
361 {
362 	*dest = *src;
363 	if (dest->grh.sgid_attr)
364 		rdma_hold_gid_attr(dest->grh.sgid_attr);
365 }
366 EXPORT_SYMBOL(rdma_copy_ah_attr);
367 
368 /**
369  * rdma_replace_ah_attr - Replace valid ah_attr with new new one.
370  * @old:        Pointer to existing ah_attr which needs to be replaced.
371  *              old is assumed to be valid or zero'd
372  * @new:        Pointer to the new ah_attr.
373  *
374  * rdma_replace_ah_attr() first releases any reference in the old ah_attr if
375  * old the ah_attr is valid; after that it copies the new attribute and holds
376  * the reference to the replaced ah_attr.
377  */
rdma_replace_ah_attr(struct rdma_ah_attr * old,const struct rdma_ah_attr * new)378 void rdma_replace_ah_attr(struct rdma_ah_attr *old,
379 			  const struct rdma_ah_attr *new)
380 {
381 	rdma_destroy_ah_attr(old);
382 	*old = *new;
383 	if (old->grh.sgid_attr)
384 		rdma_hold_gid_attr(old->grh.sgid_attr);
385 }
386 EXPORT_SYMBOL(rdma_replace_ah_attr);
387 
388 /**
389  * rdma_move_ah_attr - Move ah_attr pointed by source to destination.
390  * @dest:       Pointer to destination ah_attr to copy to.
391  *              dest is assumed to be valid or zero'd
392  * @src:        Pointer to the new ah_attr.
393  *
394  * rdma_move_ah_attr() first releases any reference in the destination ah_attr
395  * if it is valid. This also transfers ownership of internal references from
396  * src to dest, making src invalid in the process. No new reference of the src
397  * ah_attr is taken.
398  */
rdma_move_ah_attr(struct rdma_ah_attr * dest,struct rdma_ah_attr * src)399 void rdma_move_ah_attr(struct rdma_ah_attr *dest, struct rdma_ah_attr *src)
400 {
401 	rdma_destroy_ah_attr(dest);
402 	*dest = *src;
403 	src->grh.sgid_attr = NULL;
404 }
405 EXPORT_SYMBOL(rdma_move_ah_attr);
406 
407 /*
408  * Validate that the rdma_ah_attr is valid for the device before passing it
409  * off to the driver.
410  */
rdma_check_ah_attr(struct ib_device * device,struct rdma_ah_attr * ah_attr)411 static int rdma_check_ah_attr(struct ib_device *device,
412 			      struct rdma_ah_attr *ah_attr)
413 {
414 	if (!rdma_is_port_valid(device, ah_attr->port_num))
415 		return -EINVAL;
416 
417 	if ((rdma_is_grh_required(device, ah_attr->port_num) ||
418 	     ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) &&
419 	    !(ah_attr->ah_flags & IB_AH_GRH))
420 		return -EINVAL;
421 
422 	if (ah_attr->grh.sgid_attr) {
423 		/*
424 		 * Make sure the passed sgid_attr is consistent with the
425 		 * parameters
426 		 */
427 		if (ah_attr->grh.sgid_attr->index != ah_attr->grh.sgid_index ||
428 		    ah_attr->grh.sgid_attr->port_num != ah_attr->port_num)
429 			return -EINVAL;
430 	}
431 	return 0;
432 }
433 
434 /*
435  * If the ah requires a GRH then ensure that sgid_attr pointer is filled in.
436  * On success the caller is responsible to call rdma_unfill_sgid_attr().
437  */
rdma_fill_sgid_attr(struct ib_device * device,struct rdma_ah_attr * ah_attr,const struct ib_gid_attr ** old_sgid_attr)438 static int rdma_fill_sgid_attr(struct ib_device *device,
439 			       struct rdma_ah_attr *ah_attr,
440 			       const struct ib_gid_attr **old_sgid_attr)
441 {
442 	const struct ib_gid_attr *sgid_attr;
443 	struct ib_global_route *grh;
444 	int ret;
445 
446 	*old_sgid_attr = ah_attr->grh.sgid_attr;
447 
448 	ret = rdma_check_ah_attr(device, ah_attr);
449 	if (ret)
450 		return ret;
451 
452 	if (!(ah_attr->ah_flags & IB_AH_GRH))
453 		return 0;
454 
455 	grh = rdma_ah_retrieve_grh(ah_attr);
456 	if (grh->sgid_attr)
457 		return 0;
458 
459 	sgid_attr =
460 		rdma_get_gid_attr(device, ah_attr->port_num, grh->sgid_index);
461 	if (IS_ERR(sgid_attr))
462 		return PTR_ERR(sgid_attr);
463 
464 	/* Move ownerhip of the kref into the ah_attr */
465 	grh->sgid_attr = sgid_attr;
466 	return 0;
467 }
468 
rdma_unfill_sgid_attr(struct rdma_ah_attr * ah_attr,const struct ib_gid_attr * old_sgid_attr)469 static void rdma_unfill_sgid_attr(struct rdma_ah_attr *ah_attr,
470 				  const struct ib_gid_attr *old_sgid_attr)
471 {
472 	/*
473 	 * Fill didn't change anything, the caller retains ownership of
474 	 * whatever it passed
475 	 */
476 	if (ah_attr->grh.sgid_attr == old_sgid_attr)
477 		return;
478 
479 	/*
480 	 * Otherwise, we need to undo what rdma_fill_sgid_attr so the caller
481 	 * doesn't see any change in the rdma_ah_attr. If we get here
482 	 * old_sgid_attr is NULL.
483 	 */
484 	rdma_destroy_ah_attr(ah_attr);
485 }
486 
487 static const struct ib_gid_attr *
rdma_update_sgid_attr(struct rdma_ah_attr * ah_attr,const struct ib_gid_attr * old_attr)488 rdma_update_sgid_attr(struct rdma_ah_attr *ah_attr,
489 		      const struct ib_gid_attr *old_attr)
490 {
491 	if (old_attr)
492 		rdma_put_gid_attr(old_attr);
493 	if (ah_attr->ah_flags & IB_AH_GRH) {
494 		rdma_hold_gid_attr(ah_attr->grh.sgid_attr);
495 		return ah_attr->grh.sgid_attr;
496 	}
497 	return NULL;
498 }
499 
_rdma_create_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,u32 flags,struct ib_udata * udata,struct net_device * xmit_slave)500 static struct ib_ah *_rdma_create_ah(struct ib_pd *pd,
501 				     struct rdma_ah_attr *ah_attr,
502 				     u32 flags,
503 				     struct ib_udata *udata,
504 				     struct net_device *xmit_slave)
505 {
506 	struct rdma_ah_init_attr init_attr = {};
507 	struct ib_device *device = pd->device;
508 	struct ib_ah *ah;
509 	int ret;
510 
511 	might_sleep_if(flags & RDMA_CREATE_AH_SLEEPABLE);
512 
513 	if (!udata && !device->ops.create_ah)
514 		return ERR_PTR(-EOPNOTSUPP);
515 
516 	ah = rdma_zalloc_drv_obj_gfp(
517 		device, ib_ah,
518 		(flags & RDMA_CREATE_AH_SLEEPABLE) ? GFP_KERNEL : GFP_ATOMIC);
519 	if (!ah)
520 		return ERR_PTR(-ENOMEM);
521 
522 	ah->device = device;
523 	ah->pd = pd;
524 	ah->type = ah_attr->type;
525 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, NULL);
526 	init_attr.ah_attr = ah_attr;
527 	init_attr.flags = flags;
528 	init_attr.xmit_slave = xmit_slave;
529 
530 	if (udata)
531 		ret = device->ops.create_user_ah(ah, &init_attr, udata);
532 	else
533 		ret = device->ops.create_ah(ah, &init_attr, NULL);
534 	if (ret) {
535 		kfree(ah);
536 		return ERR_PTR(ret);
537 	}
538 
539 	atomic_inc(&pd->usecnt);
540 	return ah;
541 }
542 
543 /**
544  * rdma_create_ah - Creates an address handle for the
545  * given address vector.
546  * @pd: The protection domain associated with the address handle.
547  * @ah_attr: The attributes of the address vector.
548  * @flags: Create address handle flags (see enum rdma_create_ah_flags).
549  *
550  * It returns 0 on success and returns appropriate error code on error.
551  * The address handle is used to reference a local or global destination
552  * in all UD QP post sends.
553  */
rdma_create_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,u32 flags)554 struct ib_ah *rdma_create_ah(struct ib_pd *pd, struct rdma_ah_attr *ah_attr,
555 			     u32 flags)
556 {
557 	const struct ib_gid_attr *old_sgid_attr;
558 	struct net_device *slave;
559 	struct ib_ah *ah;
560 	int ret;
561 
562 	ret = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
563 	if (ret)
564 		return ERR_PTR(ret);
565 	slave = rdma_lag_get_ah_roce_slave(pd->device, ah_attr,
566 					   (flags & RDMA_CREATE_AH_SLEEPABLE) ?
567 					   GFP_KERNEL : GFP_ATOMIC);
568 	if (IS_ERR(slave)) {
569 		rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
570 		return (void *)slave;
571 	}
572 	ah = _rdma_create_ah(pd, ah_attr, flags, NULL, slave);
573 	rdma_lag_put_ah_roce_slave(slave);
574 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
575 	return ah;
576 }
577 EXPORT_SYMBOL(rdma_create_ah);
578 
579 /**
580  * rdma_create_user_ah - Creates an address handle for the
581  * given address vector.
582  * It resolves destination mac address for ah attribute of RoCE type.
583  * @pd: The protection domain associated with the address handle.
584  * @ah_attr: The attributes of the address vector.
585  * @udata: pointer to user's input output buffer information need by
586  *         provider driver.
587  *
588  * It returns 0 on success and returns appropriate error code on error.
589  * The address handle is used to reference a local or global destination
590  * in all UD QP post sends.
591  */
rdma_create_user_ah(struct ib_pd * pd,struct rdma_ah_attr * ah_attr,struct ib_udata * udata)592 struct ib_ah *rdma_create_user_ah(struct ib_pd *pd,
593 				  struct rdma_ah_attr *ah_attr,
594 				  struct ib_udata *udata)
595 {
596 	const struct ib_gid_attr *old_sgid_attr;
597 	struct ib_ah *ah;
598 	int err;
599 
600 	err = rdma_fill_sgid_attr(pd->device, ah_attr, &old_sgid_attr);
601 	if (err)
602 		return ERR_PTR(err);
603 
604 	if (ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
605 		err = ib_resolve_eth_dmac(pd->device, ah_attr);
606 		if (err) {
607 			ah = ERR_PTR(err);
608 			goto out;
609 		}
610 	}
611 
612 	ah = _rdma_create_ah(pd, ah_attr, RDMA_CREATE_AH_SLEEPABLE,
613 			     udata, NULL);
614 
615 out:
616 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
617 	return ah;
618 }
619 EXPORT_SYMBOL(rdma_create_user_ah);
620 
ib_get_rdma_header_version(const union rdma_network_hdr * hdr)621 int ib_get_rdma_header_version(const union rdma_network_hdr *hdr)
622 {
623 	const struct iphdr *ip4h = (struct iphdr *)&hdr->roce4grh;
624 	struct iphdr ip4h_checked;
625 	const struct ipv6hdr *ip6h = (struct ipv6hdr *)&hdr->ibgrh;
626 
627 	/* If it's IPv6, the version must be 6, otherwise, the first
628 	 * 20 bytes (before the IPv4 header) are garbled.
629 	 */
630 	if (ip6h->version != 6)
631 		return (ip4h->version == 4) ? 4 : 0;
632 	/* version may be 6 or 4 because the first 20 bytes could be garbled */
633 
634 	/* RoCE v2 requires no options, thus header length
635 	 * must be 5 words
636 	 */
637 	if (ip4h->ihl != 5)
638 		return 6;
639 
640 	/* Verify checksum.
641 	 * We can't write on scattered buffers so we need to copy to
642 	 * temp buffer.
643 	 */
644 	memcpy(&ip4h_checked, ip4h, sizeof(ip4h_checked));
645 	ip4h_checked.check = 0;
646 	ip4h_checked.check = ip_fast_csum((u8 *)&ip4h_checked, 5);
647 	/* if IPv4 header checksum is OK, believe it */
648 	if (ip4h->check == ip4h_checked.check)
649 		return 4;
650 	return 6;
651 }
652 EXPORT_SYMBOL(ib_get_rdma_header_version);
653 
ib_get_net_type_by_grh(struct ib_device * device,u32 port_num,const struct ib_grh * grh)654 static enum rdma_network_type ib_get_net_type_by_grh(struct ib_device *device,
655 						     u32 port_num,
656 						     const struct ib_grh *grh)
657 {
658 	int grh_version;
659 
660 	if (rdma_protocol_ib(device, port_num))
661 		return RDMA_NETWORK_IB;
662 
663 	grh_version = ib_get_rdma_header_version((union rdma_network_hdr *)grh);
664 
665 	if (grh_version == 4)
666 		return RDMA_NETWORK_IPV4;
667 
668 	if (grh->next_hdr == IPPROTO_UDP)
669 		return RDMA_NETWORK_IPV6;
670 
671 	return RDMA_NETWORK_ROCE_V1;
672 }
673 
674 struct find_gid_index_context {
675 	u16 vlan_id;
676 	enum ib_gid_type gid_type;
677 };
678 
find_gid_index(const union ib_gid * gid,const struct ib_gid_attr * gid_attr,void * context)679 static bool find_gid_index(const union ib_gid *gid,
680 			   const struct ib_gid_attr *gid_attr,
681 			   void *context)
682 {
683 	struct find_gid_index_context *ctx = context;
684 	u16 vlan_id = 0xffff;
685 	int ret;
686 
687 	if (ctx->gid_type != gid_attr->gid_type)
688 		return false;
689 
690 	ret = rdma_read_gid_l2_fields(gid_attr, &vlan_id, NULL);
691 	if (ret)
692 		return false;
693 
694 	return ctx->vlan_id == vlan_id;
695 }
696 
697 static const struct ib_gid_attr *
get_sgid_attr_from_eth(struct ib_device * device,u32 port_num,u16 vlan_id,const union ib_gid * sgid,enum ib_gid_type gid_type)698 get_sgid_attr_from_eth(struct ib_device *device, u32 port_num,
699 		       u16 vlan_id, const union ib_gid *sgid,
700 		       enum ib_gid_type gid_type)
701 {
702 	struct find_gid_index_context context = {.vlan_id = vlan_id,
703 						 .gid_type = gid_type};
704 
705 	return rdma_find_gid_by_filter(device, sgid, port_num, find_gid_index,
706 				       &context);
707 }
708 
ib_get_gids_from_rdma_hdr(const union rdma_network_hdr * hdr,enum rdma_network_type net_type,union ib_gid * sgid,union ib_gid * dgid)709 int ib_get_gids_from_rdma_hdr(const union rdma_network_hdr *hdr,
710 			      enum rdma_network_type net_type,
711 			      union ib_gid *sgid, union ib_gid *dgid)
712 {
713 	struct sockaddr_in  src_in;
714 	struct sockaddr_in  dst_in;
715 	__be32 src_saddr, dst_saddr;
716 
717 	if (!sgid || !dgid)
718 		return -EINVAL;
719 
720 	if (net_type == RDMA_NETWORK_IPV4) {
721 		memcpy(&src_in.sin_addr.s_addr,
722 		       &hdr->roce4grh.saddr, 4);
723 		memcpy(&dst_in.sin_addr.s_addr,
724 		       &hdr->roce4grh.daddr, 4);
725 		src_saddr = src_in.sin_addr.s_addr;
726 		dst_saddr = dst_in.sin_addr.s_addr;
727 		ipv6_addr_set_v4mapped(src_saddr,
728 				       (struct in6_addr *)sgid);
729 		ipv6_addr_set_v4mapped(dst_saddr,
730 				       (struct in6_addr *)dgid);
731 		return 0;
732 	} else if (net_type == RDMA_NETWORK_IPV6 ||
733 		   net_type == RDMA_NETWORK_IB || RDMA_NETWORK_ROCE_V1) {
734 		*dgid = hdr->ibgrh.dgid;
735 		*sgid = hdr->ibgrh.sgid;
736 		return 0;
737 	} else {
738 		return -EINVAL;
739 	}
740 }
741 EXPORT_SYMBOL(ib_get_gids_from_rdma_hdr);
742 
743 /* Resolve destination mac address and hop limit for unicast destination
744  * GID entry, considering the source GID entry as well.
745  * ah_attribute must have have valid port_num, sgid_index.
746  */
ib_resolve_unicast_gid_dmac(struct ib_device * device,struct rdma_ah_attr * ah_attr)747 static int ib_resolve_unicast_gid_dmac(struct ib_device *device,
748 				       struct rdma_ah_attr *ah_attr)
749 {
750 	struct ib_global_route *grh = rdma_ah_retrieve_grh(ah_attr);
751 	const struct ib_gid_attr *sgid_attr = grh->sgid_attr;
752 	int hop_limit = 0xff;
753 	int ret = 0;
754 
755 	/* If destination is link local and source GID is RoCEv1,
756 	 * IP stack is not used.
757 	 */
758 	if (rdma_link_local_addr((struct in6_addr *)grh->dgid.raw) &&
759 	    sgid_attr->gid_type == IB_GID_TYPE_ROCE) {
760 		rdma_get_ll_mac((struct in6_addr *)grh->dgid.raw,
761 				ah_attr->roce.dmac);
762 		return ret;
763 	}
764 
765 	ret = rdma_addr_find_l2_eth_by_grh(&sgid_attr->gid, &grh->dgid,
766 					   ah_attr->roce.dmac,
767 					   sgid_attr, &hop_limit);
768 
769 	grh->hop_limit = hop_limit;
770 	return ret;
771 }
772 
773 /*
774  * This function initializes address handle attributes from the incoming packet.
775  * Incoming packet has dgid of the receiver node on which this code is
776  * getting executed and, sgid contains the GID of the sender.
777  *
778  * When resolving mac address of destination, the arrived dgid is used
779  * as sgid and, sgid is used as dgid because sgid contains destinations
780  * GID whom to respond to.
781  *
782  * On success the caller is responsible to call rdma_destroy_ah_attr on the
783  * attr.
784  */
ib_init_ah_attr_from_wc(struct ib_device * device,u32 port_num,const struct ib_wc * wc,const struct ib_grh * grh,struct rdma_ah_attr * ah_attr)785 int ib_init_ah_attr_from_wc(struct ib_device *device, u32 port_num,
786 			    const struct ib_wc *wc, const struct ib_grh *grh,
787 			    struct rdma_ah_attr *ah_attr)
788 {
789 	u32 flow_class;
790 	int ret;
791 	enum rdma_network_type net_type = RDMA_NETWORK_IB;
792 	enum ib_gid_type gid_type = IB_GID_TYPE_IB;
793 	const struct ib_gid_attr *sgid_attr;
794 	int hoplimit = 0xff;
795 	union ib_gid dgid;
796 	union ib_gid sgid;
797 
798 	might_sleep();
799 
800 	memset(ah_attr, 0, sizeof *ah_attr);
801 	ah_attr->type = rdma_ah_find_type(device, port_num);
802 	if (rdma_cap_eth_ah(device, port_num)) {
803 		if (wc->wc_flags & IB_WC_WITH_NETWORK_HDR_TYPE)
804 			net_type = wc->network_hdr_type;
805 		else
806 			net_type = ib_get_net_type_by_grh(device, port_num, grh);
807 		gid_type = ib_network_to_gid_type(net_type);
808 	}
809 	ret = ib_get_gids_from_rdma_hdr((union rdma_network_hdr *)grh, net_type,
810 					&sgid, &dgid);
811 	if (ret)
812 		return ret;
813 
814 	rdma_ah_set_sl(ah_attr, wc->sl);
815 	rdma_ah_set_port_num(ah_attr, port_num);
816 
817 	if (rdma_protocol_roce(device, port_num)) {
818 		u16 vlan_id = wc->wc_flags & IB_WC_WITH_VLAN ?
819 				wc->vlan_id : 0xffff;
820 
821 		if (!(wc->wc_flags & IB_WC_GRH))
822 			return -EPROTOTYPE;
823 
824 		sgid_attr = get_sgid_attr_from_eth(device, port_num,
825 						   vlan_id, &dgid,
826 						   gid_type);
827 		if (IS_ERR(sgid_attr))
828 			return PTR_ERR(sgid_attr);
829 
830 		flow_class = be32_to_cpu(grh->version_tclass_flow);
831 		rdma_move_grh_sgid_attr(ah_attr,
832 					&sgid,
833 					flow_class & 0xFFFFF,
834 					hoplimit,
835 					(flow_class >> 20) & 0xFF,
836 					sgid_attr);
837 
838 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
839 		if (ret)
840 			rdma_destroy_ah_attr(ah_attr);
841 
842 		return ret;
843 	} else {
844 		rdma_ah_set_dlid(ah_attr, wc->slid);
845 		rdma_ah_set_path_bits(ah_attr, wc->dlid_path_bits);
846 
847 		if ((wc->wc_flags & IB_WC_GRH) == 0)
848 			return 0;
849 
850 		if (dgid.global.interface_id !=
851 					cpu_to_be64(IB_SA_WELL_KNOWN_GUID)) {
852 			sgid_attr = rdma_find_gid_by_port(
853 				device, &dgid, IB_GID_TYPE_IB, port_num, NULL);
854 		} else
855 			sgid_attr = rdma_get_gid_attr(device, port_num, 0);
856 
857 		if (IS_ERR(sgid_attr))
858 			return PTR_ERR(sgid_attr);
859 		flow_class = be32_to_cpu(grh->version_tclass_flow);
860 		rdma_move_grh_sgid_attr(ah_attr,
861 					&sgid,
862 					flow_class & 0xFFFFF,
863 					hoplimit,
864 					(flow_class >> 20) & 0xFF,
865 					sgid_attr);
866 
867 		return 0;
868 	}
869 }
870 EXPORT_SYMBOL(ib_init_ah_attr_from_wc);
871 
872 /**
873  * rdma_move_grh_sgid_attr - Sets the sgid attribute of GRH, taking ownership
874  * of the reference
875  *
876  * @attr:	Pointer to AH attribute structure
877  * @dgid:	Destination GID
878  * @flow_label:	Flow label
879  * @hop_limit:	Hop limit
880  * @traffic_class: traffic class
881  * @sgid_attr:	Pointer to SGID attribute
882  *
883  * This takes ownership of the sgid_attr reference. The caller must ensure
884  * rdma_destroy_ah_attr() is called before destroying the rdma_ah_attr after
885  * calling this function.
886  */
rdma_move_grh_sgid_attr(struct rdma_ah_attr * attr,union ib_gid * dgid,u32 flow_label,u8 hop_limit,u8 traffic_class,const struct ib_gid_attr * sgid_attr)887 void rdma_move_grh_sgid_attr(struct rdma_ah_attr *attr, union ib_gid *dgid,
888 			     u32 flow_label, u8 hop_limit, u8 traffic_class,
889 			     const struct ib_gid_attr *sgid_attr)
890 {
891 	rdma_ah_set_grh(attr, dgid, flow_label, sgid_attr->index, hop_limit,
892 			traffic_class);
893 	attr->grh.sgid_attr = sgid_attr;
894 }
895 EXPORT_SYMBOL(rdma_move_grh_sgid_attr);
896 
897 /**
898  * rdma_destroy_ah_attr - Release reference to SGID attribute of
899  * ah attribute.
900  * @ah_attr: Pointer to ah attribute
901  *
902  * Release reference to the SGID attribute of the ah attribute if it is
903  * non NULL. It is safe to call this multiple times, and safe to call it on
904  * a zero initialized ah_attr.
905  */
rdma_destroy_ah_attr(struct rdma_ah_attr * ah_attr)906 void rdma_destroy_ah_attr(struct rdma_ah_attr *ah_attr)
907 {
908 	if (ah_attr->grh.sgid_attr) {
909 		rdma_put_gid_attr(ah_attr->grh.sgid_attr);
910 		ah_attr->grh.sgid_attr = NULL;
911 	}
912 }
913 EXPORT_SYMBOL(rdma_destroy_ah_attr);
914 
ib_create_ah_from_wc(struct ib_pd * pd,const struct ib_wc * wc,const struct ib_grh * grh,u32 port_num)915 struct ib_ah *ib_create_ah_from_wc(struct ib_pd *pd, const struct ib_wc *wc,
916 				   const struct ib_grh *grh, u32 port_num)
917 {
918 	struct rdma_ah_attr ah_attr;
919 	struct ib_ah *ah;
920 	int ret;
921 
922 	ret = ib_init_ah_attr_from_wc(pd->device, port_num, wc, grh, &ah_attr);
923 	if (ret)
924 		return ERR_PTR(ret);
925 
926 	ah = rdma_create_ah(pd, &ah_attr, RDMA_CREATE_AH_SLEEPABLE);
927 
928 	rdma_destroy_ah_attr(&ah_attr);
929 	return ah;
930 }
931 EXPORT_SYMBOL(ib_create_ah_from_wc);
932 
rdma_modify_ah(struct ib_ah * ah,struct rdma_ah_attr * ah_attr)933 int rdma_modify_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
934 {
935 	const struct ib_gid_attr *old_sgid_attr;
936 	int ret;
937 
938 	if (ah->type != ah_attr->type)
939 		return -EINVAL;
940 
941 	ret = rdma_fill_sgid_attr(ah->device, ah_attr, &old_sgid_attr);
942 	if (ret)
943 		return ret;
944 
945 	ret = ah->device->ops.modify_ah ?
946 		ah->device->ops.modify_ah(ah, ah_attr) :
947 		-EOPNOTSUPP;
948 
949 	ah->sgid_attr = rdma_update_sgid_attr(ah_attr, ah->sgid_attr);
950 	rdma_unfill_sgid_attr(ah_attr, old_sgid_attr);
951 	return ret;
952 }
953 EXPORT_SYMBOL(rdma_modify_ah);
954 
rdma_query_ah(struct ib_ah * ah,struct rdma_ah_attr * ah_attr)955 int rdma_query_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr)
956 {
957 	ah_attr->grh.sgid_attr = NULL;
958 
959 	return ah->device->ops.query_ah ?
960 		ah->device->ops.query_ah(ah, ah_attr) :
961 		-EOPNOTSUPP;
962 }
963 EXPORT_SYMBOL(rdma_query_ah);
964 
rdma_destroy_ah_user(struct ib_ah * ah,u32 flags,struct ib_udata * udata)965 int rdma_destroy_ah_user(struct ib_ah *ah, u32 flags, struct ib_udata *udata)
966 {
967 	const struct ib_gid_attr *sgid_attr = ah->sgid_attr;
968 	struct ib_pd *pd;
969 	int ret;
970 
971 	might_sleep_if(flags & RDMA_DESTROY_AH_SLEEPABLE);
972 
973 	pd = ah->pd;
974 
975 	ret = ah->device->ops.destroy_ah(ah, flags);
976 	if (ret)
977 		return ret;
978 
979 	atomic_dec(&pd->usecnt);
980 	if (sgid_attr)
981 		rdma_put_gid_attr(sgid_attr);
982 
983 	kfree(ah);
984 	return ret;
985 }
986 EXPORT_SYMBOL(rdma_destroy_ah_user);
987 
988 /* Shared receive queues */
989 
990 /**
991  * ib_create_srq_user - Creates a SRQ associated with the specified protection
992  *   domain.
993  * @pd: The protection domain associated with the SRQ.
994  * @srq_init_attr: A list of initial attributes required to create the
995  *   SRQ.  If SRQ creation succeeds, then the attributes are updated to
996  *   the actual capabilities of the created SRQ.
997  * @uobject: uobject pointer if this is not a kernel SRQ
998  * @udata: udata pointer if this is not a kernel SRQ
999  *
1000  * srq_attr->max_wr and srq_attr->max_sge are read the determine the
1001  * requested size of the SRQ, and set to the actual values allocated
1002  * on return.  If ib_create_srq() succeeds, then max_wr and max_sge
1003  * will always be at least as large as the requested values.
1004  */
ib_create_srq_user(struct ib_pd * pd,struct ib_srq_init_attr * srq_init_attr,struct ib_usrq_object * uobject,struct ib_udata * udata)1005 struct ib_srq *ib_create_srq_user(struct ib_pd *pd,
1006 				  struct ib_srq_init_attr *srq_init_attr,
1007 				  struct ib_usrq_object *uobject,
1008 				  struct ib_udata *udata)
1009 {
1010 	struct ib_srq *srq;
1011 	int ret;
1012 
1013 	srq = rdma_zalloc_drv_obj(pd->device, ib_srq);
1014 	if (!srq)
1015 		return ERR_PTR(-ENOMEM);
1016 
1017 	srq->device = pd->device;
1018 	srq->pd = pd;
1019 	srq->event_handler = srq_init_attr->event_handler;
1020 	srq->srq_context = srq_init_attr->srq_context;
1021 	srq->srq_type = srq_init_attr->srq_type;
1022 	srq->uobject = uobject;
1023 
1024 	if (ib_srq_has_cq(srq->srq_type)) {
1025 		srq->ext.cq = srq_init_attr->ext.cq;
1026 		atomic_inc(&srq->ext.cq->usecnt);
1027 	}
1028 	if (srq->srq_type == IB_SRQT_XRC) {
1029 		srq->ext.xrc.xrcd = srq_init_attr->ext.xrc.xrcd;
1030 		if (srq->ext.xrc.xrcd)
1031 			atomic_inc(&srq->ext.xrc.xrcd->usecnt);
1032 	}
1033 	atomic_inc(&pd->usecnt);
1034 
1035 	rdma_restrack_new(&srq->res, RDMA_RESTRACK_SRQ);
1036 	rdma_restrack_parent_name(&srq->res, &pd->res);
1037 
1038 	ret = pd->device->ops.create_srq(srq, srq_init_attr, udata);
1039 	if (ret) {
1040 		rdma_restrack_put(&srq->res);
1041 		atomic_dec(&pd->usecnt);
1042 		if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1043 			atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1044 		if (ib_srq_has_cq(srq->srq_type))
1045 			atomic_dec(&srq->ext.cq->usecnt);
1046 		kfree(srq);
1047 		return ERR_PTR(ret);
1048 	}
1049 
1050 	rdma_restrack_add(&srq->res);
1051 
1052 	return srq;
1053 }
1054 EXPORT_SYMBOL(ib_create_srq_user);
1055 
ib_modify_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr,enum ib_srq_attr_mask srq_attr_mask)1056 int ib_modify_srq(struct ib_srq *srq,
1057 		  struct ib_srq_attr *srq_attr,
1058 		  enum ib_srq_attr_mask srq_attr_mask)
1059 {
1060 	return srq->device->ops.modify_srq ?
1061 		srq->device->ops.modify_srq(srq, srq_attr, srq_attr_mask,
1062 					    NULL) : -EOPNOTSUPP;
1063 }
1064 EXPORT_SYMBOL(ib_modify_srq);
1065 
ib_query_srq(struct ib_srq * srq,struct ib_srq_attr * srq_attr)1066 int ib_query_srq(struct ib_srq *srq,
1067 		 struct ib_srq_attr *srq_attr)
1068 {
1069 	return srq->device->ops.query_srq ?
1070 		srq->device->ops.query_srq(srq, srq_attr) : -EOPNOTSUPP;
1071 }
1072 EXPORT_SYMBOL(ib_query_srq);
1073 
ib_destroy_srq_user(struct ib_srq * srq,struct ib_udata * udata)1074 int ib_destroy_srq_user(struct ib_srq *srq, struct ib_udata *udata)
1075 {
1076 	int ret;
1077 
1078 	if (atomic_read(&srq->usecnt))
1079 		return -EBUSY;
1080 
1081 	ret = srq->device->ops.destroy_srq(srq, udata);
1082 	if (ret)
1083 		return ret;
1084 
1085 	atomic_dec(&srq->pd->usecnt);
1086 	if (srq->srq_type == IB_SRQT_XRC && srq->ext.xrc.xrcd)
1087 		atomic_dec(&srq->ext.xrc.xrcd->usecnt);
1088 	if (ib_srq_has_cq(srq->srq_type))
1089 		atomic_dec(&srq->ext.cq->usecnt);
1090 	rdma_restrack_del(&srq->res);
1091 	kfree(srq);
1092 
1093 	return ret;
1094 }
1095 EXPORT_SYMBOL(ib_destroy_srq_user);
1096 
1097 /* Queue pairs */
1098 
__ib_shared_qp_event_handler(struct ib_event * event,void * context)1099 static void __ib_shared_qp_event_handler(struct ib_event *event, void *context)
1100 {
1101 	struct ib_qp *qp = context;
1102 	unsigned long flags;
1103 
1104 	spin_lock_irqsave(&qp->device->qp_open_list_lock, flags);
1105 	list_for_each_entry(event->element.qp, &qp->open_list, open_list)
1106 		if (event->element.qp->event_handler)
1107 			event->element.qp->event_handler(event, event->element.qp->qp_context);
1108 	spin_unlock_irqrestore(&qp->device->qp_open_list_lock, flags);
1109 }
1110 
__ib_open_qp(struct ib_qp * real_qp,void (* event_handler)(struct ib_event *,void *),void * qp_context)1111 static struct ib_qp *__ib_open_qp(struct ib_qp *real_qp,
1112 				  void (*event_handler)(struct ib_event *, void *),
1113 				  void *qp_context)
1114 {
1115 	struct ib_qp *qp;
1116 	unsigned long flags;
1117 	int err;
1118 
1119 	qp = kzalloc(sizeof *qp, GFP_KERNEL);
1120 	if (!qp)
1121 		return ERR_PTR(-ENOMEM);
1122 
1123 	qp->real_qp = real_qp;
1124 	err = ib_open_shared_qp_security(qp, real_qp->device);
1125 	if (err) {
1126 		kfree(qp);
1127 		return ERR_PTR(err);
1128 	}
1129 
1130 	qp->real_qp = real_qp;
1131 	atomic_inc(&real_qp->usecnt);
1132 	qp->device = real_qp->device;
1133 	qp->event_handler = event_handler;
1134 	qp->qp_context = qp_context;
1135 	qp->qp_num = real_qp->qp_num;
1136 	qp->qp_type = real_qp->qp_type;
1137 
1138 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1139 	list_add(&qp->open_list, &real_qp->open_list);
1140 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1141 
1142 	return qp;
1143 }
1144 
ib_open_qp(struct ib_xrcd * xrcd,struct ib_qp_open_attr * qp_open_attr)1145 struct ib_qp *ib_open_qp(struct ib_xrcd *xrcd,
1146 			 struct ib_qp_open_attr *qp_open_attr)
1147 {
1148 	struct ib_qp *qp, *real_qp;
1149 
1150 	if (qp_open_attr->qp_type != IB_QPT_XRC_TGT)
1151 		return ERR_PTR(-EINVAL);
1152 
1153 	down_read(&xrcd->tgt_qps_rwsem);
1154 	real_qp = xa_load(&xrcd->tgt_qps, qp_open_attr->qp_num);
1155 	if (!real_qp) {
1156 		up_read(&xrcd->tgt_qps_rwsem);
1157 		return ERR_PTR(-EINVAL);
1158 	}
1159 	qp = __ib_open_qp(real_qp, qp_open_attr->event_handler,
1160 			  qp_open_attr->qp_context);
1161 	up_read(&xrcd->tgt_qps_rwsem);
1162 	return qp;
1163 }
1164 EXPORT_SYMBOL(ib_open_qp);
1165 
create_xrc_qp_user(struct ib_qp * qp,struct ib_qp_init_attr * qp_init_attr)1166 static struct ib_qp *create_xrc_qp_user(struct ib_qp *qp,
1167 					struct ib_qp_init_attr *qp_init_attr)
1168 {
1169 	struct ib_qp *real_qp = qp;
1170 	int err;
1171 
1172 	qp->event_handler = __ib_shared_qp_event_handler;
1173 	qp->qp_context = qp;
1174 	qp->pd = NULL;
1175 	qp->send_cq = qp->recv_cq = NULL;
1176 	qp->srq = NULL;
1177 	qp->xrcd = qp_init_attr->xrcd;
1178 	atomic_inc(&qp_init_attr->xrcd->usecnt);
1179 	INIT_LIST_HEAD(&qp->open_list);
1180 
1181 	qp = __ib_open_qp(real_qp, qp_init_attr->event_handler,
1182 			  qp_init_attr->qp_context);
1183 	if (IS_ERR(qp))
1184 		return qp;
1185 
1186 	err = xa_err(xa_store(&qp_init_attr->xrcd->tgt_qps, real_qp->qp_num,
1187 			      real_qp, GFP_KERNEL));
1188 	if (err) {
1189 		ib_close_qp(qp);
1190 		return ERR_PTR(err);
1191 	}
1192 	return qp;
1193 }
1194 
create_qp(struct ib_device * dev,struct ib_pd * pd,struct ib_qp_init_attr * attr,struct ib_udata * udata,struct ib_uqp_object * uobj,const char * caller)1195 static struct ib_qp *create_qp(struct ib_device *dev, struct ib_pd *pd,
1196 			       struct ib_qp_init_attr *attr,
1197 			       struct ib_udata *udata,
1198 			       struct ib_uqp_object *uobj, const char *caller)
1199 {
1200 	struct ib_udata dummy = {};
1201 	struct ib_qp *qp;
1202 	int ret;
1203 
1204 	if (!dev->ops.create_qp)
1205 		return ERR_PTR(-EOPNOTSUPP);
1206 
1207 	qp = rdma_zalloc_drv_obj_numa(dev, ib_qp);
1208 	if (!qp)
1209 		return ERR_PTR(-ENOMEM);
1210 
1211 	qp->device = dev;
1212 	qp->pd = pd;
1213 	qp->uobject = uobj;
1214 	qp->real_qp = qp;
1215 
1216 	qp->qp_type = attr->qp_type;
1217 	qp->rwq_ind_tbl = attr->rwq_ind_tbl;
1218 	qp->srq = attr->srq;
1219 	qp->event_handler = attr->event_handler;
1220 	qp->port = attr->port_num;
1221 	qp->qp_context = attr->qp_context;
1222 
1223 	spin_lock_init(&qp->mr_lock);
1224 	INIT_LIST_HEAD(&qp->rdma_mrs);
1225 	INIT_LIST_HEAD(&qp->sig_mrs);
1226 
1227 	qp->send_cq = attr->send_cq;
1228 	qp->recv_cq = attr->recv_cq;
1229 
1230 	rdma_restrack_new(&qp->res, RDMA_RESTRACK_QP);
1231 	WARN_ONCE(!udata && !caller, "Missing kernel QP owner");
1232 	rdma_restrack_set_name(&qp->res, udata ? NULL : caller);
1233 	ret = dev->ops.create_qp(qp, attr, udata);
1234 	if (ret)
1235 		goto err_create;
1236 
1237 	/*
1238 	 * TODO: The mlx4 internally overwrites send_cq and recv_cq.
1239 	 * Unfortunately, it is not an easy task to fix that driver.
1240 	 */
1241 	qp->send_cq = attr->send_cq;
1242 	qp->recv_cq = attr->recv_cq;
1243 
1244 	ret = ib_create_qp_security(qp, dev);
1245 	if (ret)
1246 		goto err_security;
1247 
1248 	rdma_restrack_add(&qp->res);
1249 	return qp;
1250 
1251 err_security:
1252 	qp->device->ops.destroy_qp(qp, udata ? &dummy : NULL);
1253 err_create:
1254 	rdma_restrack_put(&qp->res);
1255 	kfree(qp);
1256 	return ERR_PTR(ret);
1257 
1258 }
1259 
1260 /**
1261  * ib_create_qp_user - Creates a QP associated with the specified protection
1262  *   domain.
1263  * @dev: IB device
1264  * @pd: The protection domain associated with the QP.
1265  * @attr: A list of initial attributes required to create the
1266  *   QP.  If QP creation succeeds, then the attributes are updated to
1267  *   the actual capabilities of the created QP.
1268  * @udata: User data
1269  * @uobj: uverbs obect
1270  * @caller: caller's build-time module name
1271  */
ib_create_qp_user(struct ib_device * dev,struct ib_pd * pd,struct ib_qp_init_attr * attr,struct ib_udata * udata,struct ib_uqp_object * uobj,const char * caller)1272 struct ib_qp *ib_create_qp_user(struct ib_device *dev, struct ib_pd *pd,
1273 				struct ib_qp_init_attr *attr,
1274 				struct ib_udata *udata,
1275 				struct ib_uqp_object *uobj, const char *caller)
1276 {
1277 	struct ib_qp *qp, *xrc_qp;
1278 
1279 	if (attr->qp_type == IB_QPT_XRC_TGT)
1280 		qp = create_qp(dev, pd, attr, NULL, NULL, caller);
1281 	else
1282 		qp = create_qp(dev, pd, attr, udata, uobj, NULL);
1283 	if (attr->qp_type != IB_QPT_XRC_TGT || IS_ERR(qp))
1284 		return qp;
1285 
1286 	xrc_qp = create_xrc_qp_user(qp, attr);
1287 	if (IS_ERR(xrc_qp)) {
1288 		ib_destroy_qp(qp);
1289 		return xrc_qp;
1290 	}
1291 
1292 	xrc_qp->uobject = uobj;
1293 	return xrc_qp;
1294 }
1295 EXPORT_SYMBOL(ib_create_qp_user);
1296 
ib_qp_usecnt_inc(struct ib_qp * qp)1297 void ib_qp_usecnt_inc(struct ib_qp *qp)
1298 {
1299 	if (qp->pd)
1300 		atomic_inc(&qp->pd->usecnt);
1301 	if (qp->send_cq)
1302 		atomic_inc(&qp->send_cq->usecnt);
1303 	if (qp->recv_cq)
1304 		atomic_inc(&qp->recv_cq->usecnt);
1305 	if (qp->srq)
1306 		atomic_inc(&qp->srq->usecnt);
1307 	if (qp->rwq_ind_tbl)
1308 		atomic_inc(&qp->rwq_ind_tbl->usecnt);
1309 }
1310 EXPORT_SYMBOL(ib_qp_usecnt_inc);
1311 
ib_qp_usecnt_dec(struct ib_qp * qp)1312 void ib_qp_usecnt_dec(struct ib_qp *qp)
1313 {
1314 	if (qp->rwq_ind_tbl)
1315 		atomic_dec(&qp->rwq_ind_tbl->usecnt);
1316 	if (qp->srq)
1317 		atomic_dec(&qp->srq->usecnt);
1318 	if (qp->recv_cq)
1319 		atomic_dec(&qp->recv_cq->usecnt);
1320 	if (qp->send_cq)
1321 		atomic_dec(&qp->send_cq->usecnt);
1322 	if (qp->pd)
1323 		atomic_dec(&qp->pd->usecnt);
1324 }
1325 EXPORT_SYMBOL(ib_qp_usecnt_dec);
1326 
ib_create_qp_kernel(struct ib_pd * pd,struct ib_qp_init_attr * qp_init_attr,const char * caller)1327 struct ib_qp *ib_create_qp_kernel(struct ib_pd *pd,
1328 				  struct ib_qp_init_attr *qp_init_attr,
1329 				  const char *caller)
1330 {
1331 	struct ib_device *device = pd->device;
1332 	struct ib_qp *qp;
1333 	int ret;
1334 
1335 	/*
1336 	 * If the callers is using the RDMA API calculate the resources
1337 	 * needed for the RDMA READ/WRITE operations.
1338 	 *
1339 	 * Note that these callers need to pass in a port number.
1340 	 */
1341 	if (qp_init_attr->cap.max_rdma_ctxs)
1342 		rdma_rw_init_qp(device, qp_init_attr);
1343 
1344 	qp = create_qp(device, pd, qp_init_attr, NULL, NULL, caller);
1345 	if (IS_ERR(qp))
1346 		return qp;
1347 
1348 	ib_qp_usecnt_inc(qp);
1349 
1350 	if (qp_init_attr->cap.max_rdma_ctxs) {
1351 		ret = rdma_rw_init_mrs(qp, qp_init_attr);
1352 		if (ret)
1353 			goto err;
1354 	}
1355 
1356 	/*
1357 	 * Note: all hw drivers guarantee that max_send_sge is lower than
1358 	 * the device RDMA WRITE SGE limit but not all hw drivers ensure that
1359 	 * max_send_sge <= max_sge_rd.
1360 	 */
1361 	qp->max_write_sge = qp_init_attr->cap.max_send_sge;
1362 	qp->max_read_sge = min_t(u32, qp_init_attr->cap.max_send_sge,
1363 				 device->attrs.max_sge_rd);
1364 	if (qp_init_attr->create_flags & IB_QP_CREATE_INTEGRITY_EN)
1365 		qp->integrity_en = true;
1366 
1367 	return qp;
1368 
1369 err:
1370 	ib_destroy_qp(qp);
1371 	return ERR_PTR(ret);
1372 
1373 }
1374 EXPORT_SYMBOL(ib_create_qp_kernel);
1375 
1376 static const struct {
1377 	int			valid;
1378 	enum ib_qp_attr_mask	req_param[IB_QPT_MAX];
1379 	enum ib_qp_attr_mask	opt_param[IB_QPT_MAX];
1380 } qp_state_table[IB_QPS_ERR + 1][IB_QPS_ERR + 1] = {
1381 	[IB_QPS_RESET] = {
1382 		[IB_QPS_RESET] = { .valid = 1 },
1383 		[IB_QPS_INIT]  = {
1384 			.valid = 1,
1385 			.req_param = {
1386 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1387 						IB_QP_PORT			|
1388 						IB_QP_QKEY),
1389 				[IB_QPT_RAW_PACKET] = IB_QP_PORT,
1390 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1391 						IB_QP_PORT			|
1392 						IB_QP_ACCESS_FLAGS),
1393 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1394 						IB_QP_PORT			|
1395 						IB_QP_ACCESS_FLAGS),
1396 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1397 						IB_QP_PORT			|
1398 						IB_QP_ACCESS_FLAGS),
1399 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1400 						IB_QP_PORT			|
1401 						IB_QP_ACCESS_FLAGS),
1402 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1403 						IB_QP_QKEY),
1404 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1405 						IB_QP_QKEY),
1406 			}
1407 		},
1408 	},
1409 	[IB_QPS_INIT]  = {
1410 		[IB_QPS_RESET] = { .valid = 1 },
1411 		[IB_QPS_ERR] =   { .valid = 1 },
1412 		[IB_QPS_INIT]  = {
1413 			.valid = 1,
1414 			.opt_param = {
1415 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1416 						IB_QP_PORT			|
1417 						IB_QP_QKEY),
1418 				[IB_QPT_UC]  = (IB_QP_PKEY_INDEX		|
1419 						IB_QP_PORT			|
1420 						IB_QP_ACCESS_FLAGS),
1421 				[IB_QPT_RC]  = (IB_QP_PKEY_INDEX		|
1422 						IB_QP_PORT			|
1423 						IB_QP_ACCESS_FLAGS),
1424 				[IB_QPT_XRC_INI] = (IB_QP_PKEY_INDEX		|
1425 						IB_QP_PORT			|
1426 						IB_QP_ACCESS_FLAGS),
1427 				[IB_QPT_XRC_TGT] = (IB_QP_PKEY_INDEX		|
1428 						IB_QP_PORT			|
1429 						IB_QP_ACCESS_FLAGS),
1430 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1431 						IB_QP_QKEY),
1432 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1433 						IB_QP_QKEY),
1434 			}
1435 		},
1436 		[IB_QPS_RTR]   = {
1437 			.valid = 1,
1438 			.req_param = {
1439 				[IB_QPT_UC]  = (IB_QP_AV			|
1440 						IB_QP_PATH_MTU			|
1441 						IB_QP_DEST_QPN			|
1442 						IB_QP_RQ_PSN),
1443 				[IB_QPT_RC]  = (IB_QP_AV			|
1444 						IB_QP_PATH_MTU			|
1445 						IB_QP_DEST_QPN			|
1446 						IB_QP_RQ_PSN			|
1447 						IB_QP_MAX_DEST_RD_ATOMIC	|
1448 						IB_QP_MIN_RNR_TIMER),
1449 				[IB_QPT_XRC_INI] = (IB_QP_AV			|
1450 						IB_QP_PATH_MTU			|
1451 						IB_QP_DEST_QPN			|
1452 						IB_QP_RQ_PSN),
1453 				[IB_QPT_XRC_TGT] = (IB_QP_AV			|
1454 						IB_QP_PATH_MTU			|
1455 						IB_QP_DEST_QPN			|
1456 						IB_QP_RQ_PSN			|
1457 						IB_QP_MAX_DEST_RD_ATOMIC	|
1458 						IB_QP_MIN_RNR_TIMER),
1459 			},
1460 			.opt_param = {
1461 				 [IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1462 						 IB_QP_QKEY),
1463 				 [IB_QPT_UC]  = (IB_QP_ALT_PATH			|
1464 						 IB_QP_ACCESS_FLAGS		|
1465 						 IB_QP_PKEY_INDEX),
1466 				 [IB_QPT_RC]  = (IB_QP_ALT_PATH			|
1467 						 IB_QP_ACCESS_FLAGS		|
1468 						 IB_QP_PKEY_INDEX),
1469 				 [IB_QPT_XRC_INI] = (IB_QP_ALT_PATH		|
1470 						 IB_QP_ACCESS_FLAGS		|
1471 						 IB_QP_PKEY_INDEX),
1472 				 [IB_QPT_XRC_TGT] = (IB_QP_ALT_PATH		|
1473 						 IB_QP_ACCESS_FLAGS		|
1474 						 IB_QP_PKEY_INDEX),
1475 				 [IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1476 						 IB_QP_QKEY),
1477 				 [IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1478 						 IB_QP_QKEY),
1479 			 },
1480 		},
1481 	},
1482 	[IB_QPS_RTR]   = {
1483 		[IB_QPS_RESET] = { .valid = 1 },
1484 		[IB_QPS_ERR] =   { .valid = 1 },
1485 		[IB_QPS_RTS]   = {
1486 			.valid = 1,
1487 			.req_param = {
1488 				[IB_QPT_UD]  = IB_QP_SQ_PSN,
1489 				[IB_QPT_UC]  = IB_QP_SQ_PSN,
1490 				[IB_QPT_RC]  = (IB_QP_TIMEOUT			|
1491 						IB_QP_RETRY_CNT			|
1492 						IB_QP_RNR_RETRY			|
1493 						IB_QP_SQ_PSN			|
1494 						IB_QP_MAX_QP_RD_ATOMIC),
1495 				[IB_QPT_XRC_INI] = (IB_QP_TIMEOUT		|
1496 						IB_QP_RETRY_CNT			|
1497 						IB_QP_RNR_RETRY			|
1498 						IB_QP_SQ_PSN			|
1499 						IB_QP_MAX_QP_RD_ATOMIC),
1500 				[IB_QPT_XRC_TGT] = (IB_QP_TIMEOUT		|
1501 						IB_QP_SQ_PSN),
1502 				[IB_QPT_SMI] = IB_QP_SQ_PSN,
1503 				[IB_QPT_GSI] = IB_QP_SQ_PSN,
1504 			},
1505 			.opt_param = {
1506 				 [IB_QPT_UD]  = (IB_QP_CUR_STATE		|
1507 						 IB_QP_QKEY),
1508 				 [IB_QPT_UC]  = (IB_QP_CUR_STATE		|
1509 						 IB_QP_ALT_PATH			|
1510 						 IB_QP_ACCESS_FLAGS		|
1511 						 IB_QP_PATH_MIG_STATE),
1512 				 [IB_QPT_RC]  = (IB_QP_CUR_STATE		|
1513 						 IB_QP_ALT_PATH			|
1514 						 IB_QP_ACCESS_FLAGS		|
1515 						 IB_QP_MIN_RNR_TIMER		|
1516 						 IB_QP_PATH_MIG_STATE),
1517 				 [IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1518 						 IB_QP_ALT_PATH			|
1519 						 IB_QP_ACCESS_FLAGS		|
1520 						 IB_QP_PATH_MIG_STATE),
1521 				 [IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1522 						 IB_QP_ALT_PATH			|
1523 						 IB_QP_ACCESS_FLAGS		|
1524 						 IB_QP_MIN_RNR_TIMER		|
1525 						 IB_QP_PATH_MIG_STATE),
1526 				 [IB_QPT_SMI] = (IB_QP_CUR_STATE		|
1527 						 IB_QP_QKEY),
1528 				 [IB_QPT_GSI] = (IB_QP_CUR_STATE		|
1529 						 IB_QP_QKEY),
1530 				 [IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1531 			 }
1532 		}
1533 	},
1534 	[IB_QPS_RTS]   = {
1535 		[IB_QPS_RESET] = { .valid = 1 },
1536 		[IB_QPS_ERR] =   { .valid = 1 },
1537 		[IB_QPS_RTS]   = {
1538 			.valid = 1,
1539 			.opt_param = {
1540 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1541 						IB_QP_QKEY),
1542 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1543 						IB_QP_ACCESS_FLAGS		|
1544 						IB_QP_ALT_PATH			|
1545 						IB_QP_PATH_MIG_STATE),
1546 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1547 						IB_QP_ACCESS_FLAGS		|
1548 						IB_QP_ALT_PATH			|
1549 						IB_QP_PATH_MIG_STATE		|
1550 						IB_QP_MIN_RNR_TIMER),
1551 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1552 						IB_QP_ACCESS_FLAGS		|
1553 						IB_QP_ALT_PATH			|
1554 						IB_QP_PATH_MIG_STATE),
1555 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1556 						IB_QP_ACCESS_FLAGS		|
1557 						IB_QP_ALT_PATH			|
1558 						IB_QP_PATH_MIG_STATE		|
1559 						IB_QP_MIN_RNR_TIMER),
1560 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1561 						IB_QP_QKEY),
1562 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1563 						IB_QP_QKEY),
1564 				[IB_QPT_RAW_PACKET] = IB_QP_RATE_LIMIT,
1565 			}
1566 		},
1567 		[IB_QPS_SQD]   = {
1568 			.valid = 1,
1569 			.opt_param = {
1570 				[IB_QPT_UD]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1571 				[IB_QPT_UC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1572 				[IB_QPT_RC]  = IB_QP_EN_SQD_ASYNC_NOTIFY,
1573 				[IB_QPT_XRC_INI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1574 				[IB_QPT_XRC_TGT] = IB_QP_EN_SQD_ASYNC_NOTIFY, /* ??? */
1575 				[IB_QPT_SMI] = IB_QP_EN_SQD_ASYNC_NOTIFY,
1576 				[IB_QPT_GSI] = IB_QP_EN_SQD_ASYNC_NOTIFY
1577 			}
1578 		},
1579 	},
1580 	[IB_QPS_SQD]   = {
1581 		[IB_QPS_RESET] = { .valid = 1 },
1582 		[IB_QPS_ERR] =   { .valid = 1 },
1583 		[IB_QPS_RTS]   = {
1584 			.valid = 1,
1585 			.opt_param = {
1586 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1587 						IB_QP_QKEY),
1588 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1589 						IB_QP_ALT_PATH			|
1590 						IB_QP_ACCESS_FLAGS		|
1591 						IB_QP_PATH_MIG_STATE),
1592 				[IB_QPT_RC]  = (IB_QP_CUR_STATE			|
1593 						IB_QP_ALT_PATH			|
1594 						IB_QP_ACCESS_FLAGS		|
1595 						IB_QP_MIN_RNR_TIMER		|
1596 						IB_QP_PATH_MIG_STATE),
1597 				[IB_QPT_XRC_INI] = (IB_QP_CUR_STATE		|
1598 						IB_QP_ALT_PATH			|
1599 						IB_QP_ACCESS_FLAGS		|
1600 						IB_QP_PATH_MIG_STATE),
1601 				[IB_QPT_XRC_TGT] = (IB_QP_CUR_STATE		|
1602 						IB_QP_ALT_PATH			|
1603 						IB_QP_ACCESS_FLAGS		|
1604 						IB_QP_MIN_RNR_TIMER		|
1605 						IB_QP_PATH_MIG_STATE),
1606 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1607 						IB_QP_QKEY),
1608 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1609 						IB_QP_QKEY),
1610 			}
1611 		},
1612 		[IB_QPS_SQD]   = {
1613 			.valid = 1,
1614 			.opt_param = {
1615 				[IB_QPT_UD]  = (IB_QP_PKEY_INDEX		|
1616 						IB_QP_QKEY),
1617 				[IB_QPT_UC]  = (IB_QP_AV			|
1618 						IB_QP_ALT_PATH			|
1619 						IB_QP_ACCESS_FLAGS		|
1620 						IB_QP_PKEY_INDEX		|
1621 						IB_QP_PATH_MIG_STATE),
1622 				[IB_QPT_RC]  = (IB_QP_PORT			|
1623 						IB_QP_AV			|
1624 						IB_QP_TIMEOUT			|
1625 						IB_QP_RETRY_CNT			|
1626 						IB_QP_RNR_RETRY			|
1627 						IB_QP_MAX_QP_RD_ATOMIC		|
1628 						IB_QP_MAX_DEST_RD_ATOMIC	|
1629 						IB_QP_ALT_PATH			|
1630 						IB_QP_ACCESS_FLAGS		|
1631 						IB_QP_PKEY_INDEX		|
1632 						IB_QP_MIN_RNR_TIMER		|
1633 						IB_QP_PATH_MIG_STATE),
1634 				[IB_QPT_XRC_INI] = (IB_QP_PORT			|
1635 						IB_QP_AV			|
1636 						IB_QP_TIMEOUT			|
1637 						IB_QP_RETRY_CNT			|
1638 						IB_QP_RNR_RETRY			|
1639 						IB_QP_MAX_QP_RD_ATOMIC		|
1640 						IB_QP_ALT_PATH			|
1641 						IB_QP_ACCESS_FLAGS		|
1642 						IB_QP_PKEY_INDEX		|
1643 						IB_QP_PATH_MIG_STATE),
1644 				[IB_QPT_XRC_TGT] = (IB_QP_PORT			|
1645 						IB_QP_AV			|
1646 						IB_QP_TIMEOUT			|
1647 						IB_QP_MAX_DEST_RD_ATOMIC	|
1648 						IB_QP_ALT_PATH			|
1649 						IB_QP_ACCESS_FLAGS		|
1650 						IB_QP_PKEY_INDEX		|
1651 						IB_QP_MIN_RNR_TIMER		|
1652 						IB_QP_PATH_MIG_STATE),
1653 				[IB_QPT_SMI] = (IB_QP_PKEY_INDEX		|
1654 						IB_QP_QKEY),
1655 				[IB_QPT_GSI] = (IB_QP_PKEY_INDEX		|
1656 						IB_QP_QKEY),
1657 			}
1658 		}
1659 	},
1660 	[IB_QPS_SQE]   = {
1661 		[IB_QPS_RESET] = { .valid = 1 },
1662 		[IB_QPS_ERR] =   { .valid = 1 },
1663 		[IB_QPS_RTS]   = {
1664 			.valid = 1,
1665 			.opt_param = {
1666 				[IB_QPT_UD]  = (IB_QP_CUR_STATE			|
1667 						IB_QP_QKEY),
1668 				[IB_QPT_UC]  = (IB_QP_CUR_STATE			|
1669 						IB_QP_ACCESS_FLAGS),
1670 				[IB_QPT_SMI] = (IB_QP_CUR_STATE			|
1671 						IB_QP_QKEY),
1672 				[IB_QPT_GSI] = (IB_QP_CUR_STATE			|
1673 						IB_QP_QKEY),
1674 			}
1675 		}
1676 	},
1677 	[IB_QPS_ERR] = {
1678 		[IB_QPS_RESET] = { .valid = 1 },
1679 		[IB_QPS_ERR] =   { .valid = 1 }
1680 	}
1681 };
1682 
ib_modify_qp_is_ok(enum ib_qp_state cur_state,enum ib_qp_state next_state,enum ib_qp_type type,enum ib_qp_attr_mask mask)1683 bool ib_modify_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state next_state,
1684 			enum ib_qp_type type, enum ib_qp_attr_mask mask)
1685 {
1686 	enum ib_qp_attr_mask req_param, opt_param;
1687 
1688 	if (mask & IB_QP_CUR_STATE  &&
1689 	    cur_state != IB_QPS_RTR && cur_state != IB_QPS_RTS &&
1690 	    cur_state != IB_QPS_SQD && cur_state != IB_QPS_SQE)
1691 		return false;
1692 
1693 	if (!qp_state_table[cur_state][next_state].valid)
1694 		return false;
1695 
1696 	req_param = qp_state_table[cur_state][next_state].req_param[type];
1697 	opt_param = qp_state_table[cur_state][next_state].opt_param[type];
1698 
1699 	if ((mask & req_param) != req_param)
1700 		return false;
1701 
1702 	if (mask & ~(req_param | opt_param | IB_QP_STATE))
1703 		return false;
1704 
1705 	return true;
1706 }
1707 EXPORT_SYMBOL(ib_modify_qp_is_ok);
1708 
1709 /**
1710  * ib_resolve_eth_dmac - Resolve destination mac address
1711  * @device:		Device to consider
1712  * @ah_attr:		address handle attribute which describes the
1713  *			source and destination parameters
1714  * ib_resolve_eth_dmac() resolves destination mac address and L3 hop limit It
1715  * returns 0 on success or appropriate error code. It initializes the
1716  * necessary ah_attr fields when call is successful.
1717  */
ib_resolve_eth_dmac(struct ib_device * device,struct rdma_ah_attr * ah_attr)1718 static int ib_resolve_eth_dmac(struct ib_device *device,
1719 			       struct rdma_ah_attr *ah_attr)
1720 {
1721 	int ret = 0;
1722 
1723 	if (rdma_is_multicast_addr((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1724 		if (ipv6_addr_v4mapped((struct in6_addr *)ah_attr->grh.dgid.raw)) {
1725 			__be32 addr = 0;
1726 
1727 			memcpy(&addr, ah_attr->grh.dgid.raw + 12, 4);
1728 			ip_eth_mc_map(addr, (char *)ah_attr->roce.dmac);
1729 		} else {
1730 			ipv6_eth_mc_map((struct in6_addr *)ah_attr->grh.dgid.raw,
1731 					(char *)ah_attr->roce.dmac);
1732 		}
1733 	} else {
1734 		ret = ib_resolve_unicast_gid_dmac(device, ah_attr);
1735 	}
1736 	return ret;
1737 }
1738 
is_qp_type_connected(const struct ib_qp * qp)1739 static bool is_qp_type_connected(const struct ib_qp *qp)
1740 {
1741 	return (qp->qp_type == IB_QPT_UC ||
1742 		qp->qp_type == IB_QPT_RC ||
1743 		qp->qp_type == IB_QPT_XRC_INI ||
1744 		qp->qp_type == IB_QPT_XRC_TGT);
1745 }
1746 
1747 /*
1748  * IB core internal function to perform QP attributes modification.
1749  */
_ib_modify_qp(struct ib_qp * qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1750 static int _ib_modify_qp(struct ib_qp *qp, struct ib_qp_attr *attr,
1751 			 int attr_mask, struct ib_udata *udata)
1752 {
1753 	u32 port = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1754 	const struct ib_gid_attr *old_sgid_attr_av;
1755 	const struct ib_gid_attr *old_sgid_attr_alt_av;
1756 	int ret;
1757 
1758 	attr->xmit_slave = NULL;
1759 	if (attr_mask & IB_QP_AV) {
1760 		ret = rdma_fill_sgid_attr(qp->device, &attr->ah_attr,
1761 					  &old_sgid_attr_av);
1762 		if (ret)
1763 			return ret;
1764 
1765 		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE &&
1766 		    is_qp_type_connected(qp)) {
1767 			struct net_device *slave;
1768 
1769 			/*
1770 			 * If the user provided the qp_attr then we have to
1771 			 * resolve it. Kerne users have to provide already
1772 			 * resolved rdma_ah_attr's.
1773 			 */
1774 			if (udata) {
1775 				ret = ib_resolve_eth_dmac(qp->device,
1776 							  &attr->ah_attr);
1777 				if (ret)
1778 					goto out_av;
1779 			}
1780 			slave = rdma_lag_get_ah_roce_slave(qp->device,
1781 							   &attr->ah_attr,
1782 							   GFP_KERNEL);
1783 			if (IS_ERR(slave)) {
1784 				ret = PTR_ERR(slave);
1785 				goto out_av;
1786 			}
1787 			attr->xmit_slave = slave;
1788 		}
1789 	}
1790 	if (attr_mask & IB_QP_ALT_PATH) {
1791 		/*
1792 		 * FIXME: This does not track the migration state, so if the
1793 		 * user loads a new alternate path after the HW has migrated
1794 		 * from primary->alternate we will keep the wrong
1795 		 * references. This is OK for IB because the reference
1796 		 * counting does not serve any functional purpose.
1797 		 */
1798 		ret = rdma_fill_sgid_attr(qp->device, &attr->alt_ah_attr,
1799 					  &old_sgid_attr_alt_av);
1800 		if (ret)
1801 			goto out_av;
1802 
1803 		/*
1804 		 * Today the core code can only handle alternate paths and APM
1805 		 * for IB. Ban them in roce mode.
1806 		 */
1807 		if (!(rdma_protocol_ib(qp->device,
1808 				       attr->alt_ah_attr.port_num) &&
1809 		      rdma_protocol_ib(qp->device, port))) {
1810 			ret = -EINVAL;
1811 			goto out;
1812 		}
1813 	}
1814 
1815 	if (rdma_ib_or_roce(qp->device, port)) {
1816 		if (attr_mask & IB_QP_RQ_PSN && attr->rq_psn & ~0xffffff) {
1817 			dev_warn(&qp->device->dev,
1818 				 "%s rq_psn overflow, masking to 24 bits\n",
1819 				 __func__);
1820 			attr->rq_psn &= 0xffffff;
1821 		}
1822 
1823 		if (attr_mask & IB_QP_SQ_PSN && attr->sq_psn & ~0xffffff) {
1824 			dev_warn(&qp->device->dev,
1825 				 " %s sq_psn overflow, masking to 24 bits\n",
1826 				 __func__);
1827 			attr->sq_psn &= 0xffffff;
1828 		}
1829 	}
1830 
1831 	/*
1832 	 * Bind this qp to a counter automatically based on the rdma counter
1833 	 * rules. This only set in RST2INIT with port specified
1834 	 */
1835 	if (!qp->counter && (attr_mask & IB_QP_PORT) &&
1836 	    ((attr_mask & IB_QP_STATE) && attr->qp_state == IB_QPS_INIT))
1837 		rdma_counter_bind_qp_auto(qp, attr->port_num);
1838 
1839 	ret = ib_security_modify_qp(qp, attr, attr_mask, udata);
1840 	if (ret)
1841 		goto out;
1842 
1843 	if (attr_mask & IB_QP_PORT)
1844 		qp->port = attr->port_num;
1845 	if (attr_mask & IB_QP_AV)
1846 		qp->av_sgid_attr =
1847 			rdma_update_sgid_attr(&attr->ah_attr, qp->av_sgid_attr);
1848 	if (attr_mask & IB_QP_ALT_PATH)
1849 		qp->alt_path_sgid_attr = rdma_update_sgid_attr(
1850 			&attr->alt_ah_attr, qp->alt_path_sgid_attr);
1851 
1852 out:
1853 	if (attr_mask & IB_QP_ALT_PATH)
1854 		rdma_unfill_sgid_attr(&attr->alt_ah_attr, old_sgid_attr_alt_av);
1855 out_av:
1856 	if (attr_mask & IB_QP_AV) {
1857 		rdma_lag_put_ah_roce_slave(attr->xmit_slave);
1858 		rdma_unfill_sgid_attr(&attr->ah_attr, old_sgid_attr_av);
1859 	}
1860 	return ret;
1861 }
1862 
1863 /**
1864  * ib_modify_qp_with_udata - Modifies the attributes for the specified QP.
1865  * @ib_qp: The QP to modify.
1866  * @attr: On input, specifies the QP attributes to modify.  On output,
1867  *   the current values of selected QP attributes are returned.
1868  * @attr_mask: A bit-mask used to specify which attributes of the QP
1869  *   are being modified.
1870  * @udata: pointer to user's input output buffer information
1871  *   are being modified.
1872  * It returns 0 on success and returns appropriate error code on error.
1873  */
ib_modify_qp_with_udata(struct ib_qp * ib_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)1874 int ib_modify_qp_with_udata(struct ib_qp *ib_qp, struct ib_qp_attr *attr,
1875 			    int attr_mask, struct ib_udata *udata)
1876 {
1877 	return _ib_modify_qp(ib_qp->real_qp, attr, attr_mask, udata);
1878 }
1879 EXPORT_SYMBOL(ib_modify_qp_with_udata);
1880 
ib_get_eth_speed(struct ib_device * dev,u32 port_num,u16 * speed,u8 * width)1881 int ib_get_eth_speed(struct ib_device *dev, u32 port_num, u16 *speed, u8 *width)
1882 {
1883 	int rc;
1884 	u32 netdev_speed;
1885 	struct net_device *netdev;
1886 	struct ethtool_link_ksettings lksettings;
1887 
1888 	if (rdma_port_get_link_layer(dev, port_num) != IB_LINK_LAYER_ETHERNET)
1889 		return -EINVAL;
1890 
1891 	netdev = ib_device_get_netdev(dev, port_num);
1892 	if (!netdev)
1893 		return -ENODEV;
1894 
1895 	rtnl_lock();
1896 	rc = __ethtool_get_link_ksettings(netdev, &lksettings);
1897 	rtnl_unlock();
1898 
1899 	dev_put(netdev);
1900 
1901 	if (!rc && lksettings.base.speed != (u32)SPEED_UNKNOWN) {
1902 		netdev_speed = lksettings.base.speed;
1903 	} else {
1904 		netdev_speed = SPEED_1000;
1905 		pr_warn("%s speed is unknown, defaulting to %u\n", netdev->name,
1906 			netdev_speed);
1907 	}
1908 
1909 	if (netdev_speed <= SPEED_1000) {
1910 		*width = IB_WIDTH_1X;
1911 		*speed = IB_SPEED_SDR;
1912 	} else if (netdev_speed <= SPEED_10000) {
1913 		*width = IB_WIDTH_1X;
1914 		*speed = IB_SPEED_FDR10;
1915 	} else if (netdev_speed <= SPEED_20000) {
1916 		*width = IB_WIDTH_4X;
1917 		*speed = IB_SPEED_DDR;
1918 	} else if (netdev_speed <= SPEED_25000) {
1919 		*width = IB_WIDTH_1X;
1920 		*speed = IB_SPEED_EDR;
1921 	} else if (netdev_speed <= SPEED_40000) {
1922 		*width = IB_WIDTH_4X;
1923 		*speed = IB_SPEED_FDR10;
1924 	} else {
1925 		*width = IB_WIDTH_4X;
1926 		*speed = IB_SPEED_EDR;
1927 	}
1928 
1929 	return 0;
1930 }
1931 EXPORT_SYMBOL(ib_get_eth_speed);
1932 
ib_modify_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask)1933 int ib_modify_qp(struct ib_qp *qp,
1934 		 struct ib_qp_attr *qp_attr,
1935 		 int qp_attr_mask)
1936 {
1937 	return _ib_modify_qp(qp->real_qp, qp_attr, qp_attr_mask, NULL);
1938 }
1939 EXPORT_SYMBOL(ib_modify_qp);
1940 
ib_query_qp(struct ib_qp * qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)1941 int ib_query_qp(struct ib_qp *qp,
1942 		struct ib_qp_attr *qp_attr,
1943 		int qp_attr_mask,
1944 		struct ib_qp_init_attr *qp_init_attr)
1945 {
1946 	qp_attr->ah_attr.grh.sgid_attr = NULL;
1947 	qp_attr->alt_ah_attr.grh.sgid_attr = NULL;
1948 
1949 	return qp->device->ops.query_qp ?
1950 		qp->device->ops.query_qp(qp->real_qp, qp_attr, qp_attr_mask,
1951 					 qp_init_attr) : -EOPNOTSUPP;
1952 }
1953 EXPORT_SYMBOL(ib_query_qp);
1954 
ib_close_qp(struct ib_qp * qp)1955 int ib_close_qp(struct ib_qp *qp)
1956 {
1957 	struct ib_qp *real_qp;
1958 	unsigned long flags;
1959 
1960 	real_qp = qp->real_qp;
1961 	if (real_qp == qp)
1962 		return -EINVAL;
1963 
1964 	spin_lock_irqsave(&real_qp->device->qp_open_list_lock, flags);
1965 	list_del(&qp->open_list);
1966 	spin_unlock_irqrestore(&real_qp->device->qp_open_list_lock, flags);
1967 
1968 	atomic_dec(&real_qp->usecnt);
1969 	if (qp->qp_sec)
1970 		ib_close_shared_qp_security(qp->qp_sec);
1971 	kfree(qp);
1972 
1973 	return 0;
1974 }
1975 EXPORT_SYMBOL(ib_close_qp);
1976 
__ib_destroy_shared_qp(struct ib_qp * qp)1977 static int __ib_destroy_shared_qp(struct ib_qp *qp)
1978 {
1979 	struct ib_xrcd *xrcd;
1980 	struct ib_qp *real_qp;
1981 	int ret;
1982 
1983 	real_qp = qp->real_qp;
1984 	xrcd = real_qp->xrcd;
1985 	down_write(&xrcd->tgt_qps_rwsem);
1986 	ib_close_qp(qp);
1987 	if (atomic_read(&real_qp->usecnt) == 0)
1988 		xa_erase(&xrcd->tgt_qps, real_qp->qp_num);
1989 	else
1990 		real_qp = NULL;
1991 	up_write(&xrcd->tgt_qps_rwsem);
1992 
1993 	if (real_qp) {
1994 		ret = ib_destroy_qp(real_qp);
1995 		if (!ret)
1996 			atomic_dec(&xrcd->usecnt);
1997 	}
1998 
1999 	return 0;
2000 }
2001 
ib_destroy_qp_user(struct ib_qp * qp,struct ib_udata * udata)2002 int ib_destroy_qp_user(struct ib_qp *qp, struct ib_udata *udata)
2003 {
2004 	const struct ib_gid_attr *alt_path_sgid_attr = qp->alt_path_sgid_attr;
2005 	const struct ib_gid_attr *av_sgid_attr = qp->av_sgid_attr;
2006 	struct ib_qp_security *sec;
2007 	int ret;
2008 
2009 	WARN_ON_ONCE(qp->mrs_used > 0);
2010 
2011 	if (atomic_read(&qp->usecnt))
2012 		return -EBUSY;
2013 
2014 	if (qp->real_qp != qp)
2015 		return __ib_destroy_shared_qp(qp);
2016 
2017 	sec  = qp->qp_sec;
2018 	if (sec)
2019 		ib_destroy_qp_security_begin(sec);
2020 
2021 	if (!qp->uobject)
2022 		rdma_rw_cleanup_mrs(qp);
2023 
2024 	rdma_counter_unbind_qp(qp, true);
2025 	ret = qp->device->ops.destroy_qp(qp, udata);
2026 	if (ret) {
2027 		if (sec)
2028 			ib_destroy_qp_security_abort(sec);
2029 		return ret;
2030 	}
2031 
2032 	if (alt_path_sgid_attr)
2033 		rdma_put_gid_attr(alt_path_sgid_attr);
2034 	if (av_sgid_attr)
2035 		rdma_put_gid_attr(av_sgid_attr);
2036 
2037 	ib_qp_usecnt_dec(qp);
2038 	if (sec)
2039 		ib_destroy_qp_security_end(sec);
2040 
2041 	rdma_restrack_del(&qp->res);
2042 	kfree(qp);
2043 	return ret;
2044 }
2045 EXPORT_SYMBOL(ib_destroy_qp_user);
2046 
2047 /* Completion queues */
2048 
__ib_create_cq(struct ib_device * device,ib_comp_handler comp_handler,void (* event_handler)(struct ib_event *,void *),void * cq_context,const struct ib_cq_init_attr * cq_attr,const char * caller)2049 struct ib_cq *__ib_create_cq(struct ib_device *device,
2050 			     ib_comp_handler comp_handler,
2051 			     void (*event_handler)(struct ib_event *, void *),
2052 			     void *cq_context,
2053 			     const struct ib_cq_init_attr *cq_attr,
2054 			     const char *caller)
2055 {
2056 	struct ib_cq *cq;
2057 	int ret;
2058 
2059 	cq = rdma_zalloc_drv_obj(device, ib_cq);
2060 	if (!cq)
2061 		return ERR_PTR(-ENOMEM);
2062 
2063 	cq->device = device;
2064 	cq->uobject = NULL;
2065 	cq->comp_handler = comp_handler;
2066 	cq->event_handler = event_handler;
2067 	cq->cq_context = cq_context;
2068 	atomic_set(&cq->usecnt, 0);
2069 
2070 	rdma_restrack_new(&cq->res, RDMA_RESTRACK_CQ);
2071 	rdma_restrack_set_name(&cq->res, caller);
2072 
2073 	ret = device->ops.create_cq(cq, cq_attr, NULL);
2074 	if (ret) {
2075 		rdma_restrack_put(&cq->res);
2076 		kfree(cq);
2077 		return ERR_PTR(ret);
2078 	}
2079 
2080 	rdma_restrack_add(&cq->res);
2081 	return cq;
2082 }
2083 EXPORT_SYMBOL(__ib_create_cq);
2084 
rdma_set_cq_moderation(struct ib_cq * cq,u16 cq_count,u16 cq_period)2085 int rdma_set_cq_moderation(struct ib_cq *cq, u16 cq_count, u16 cq_period)
2086 {
2087 	if (cq->shared)
2088 		return -EOPNOTSUPP;
2089 
2090 	return cq->device->ops.modify_cq ?
2091 		cq->device->ops.modify_cq(cq, cq_count,
2092 					  cq_period) : -EOPNOTSUPP;
2093 }
2094 EXPORT_SYMBOL(rdma_set_cq_moderation);
2095 
ib_destroy_cq_user(struct ib_cq * cq,struct ib_udata * udata)2096 int ib_destroy_cq_user(struct ib_cq *cq, struct ib_udata *udata)
2097 {
2098 	int ret;
2099 
2100 	if (WARN_ON_ONCE(cq->shared))
2101 		return -EOPNOTSUPP;
2102 
2103 	if (atomic_read(&cq->usecnt))
2104 		return -EBUSY;
2105 
2106 	ret = cq->device->ops.destroy_cq(cq, udata);
2107 	if (ret)
2108 		return ret;
2109 
2110 	rdma_restrack_del(&cq->res);
2111 	kfree(cq);
2112 	return ret;
2113 }
2114 EXPORT_SYMBOL(ib_destroy_cq_user);
2115 
ib_resize_cq(struct ib_cq * cq,int cqe)2116 int ib_resize_cq(struct ib_cq *cq, int cqe)
2117 {
2118 	if (cq->shared)
2119 		return -EOPNOTSUPP;
2120 
2121 	return cq->device->ops.resize_cq ?
2122 		cq->device->ops.resize_cq(cq, cqe, NULL) : -EOPNOTSUPP;
2123 }
2124 EXPORT_SYMBOL(ib_resize_cq);
2125 
2126 /* Memory regions */
2127 
ib_reg_user_mr(struct ib_pd * pd,u64 start,u64 length,u64 virt_addr,int access_flags)2128 struct ib_mr *ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
2129 			     u64 virt_addr, int access_flags)
2130 {
2131 	struct ib_mr *mr;
2132 
2133 	if (access_flags & IB_ACCESS_ON_DEMAND) {
2134 		if (!(pd->device->attrs.kernel_cap_flags &
2135 		      IBK_ON_DEMAND_PAGING)) {
2136 			pr_debug("ODP support not available\n");
2137 			return ERR_PTR(-EINVAL);
2138 		}
2139 	}
2140 
2141 	mr = pd->device->ops.reg_user_mr(pd, start, length, virt_addr,
2142 					 access_flags, NULL);
2143 
2144 	if (IS_ERR(mr))
2145 		return mr;
2146 
2147 	mr->device = pd->device;
2148 	mr->type = IB_MR_TYPE_USER;
2149 	mr->pd = pd;
2150 	mr->dm = NULL;
2151 	atomic_inc(&pd->usecnt);
2152 	mr->iova =  virt_addr;
2153 	mr->length = length;
2154 
2155 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2156 	rdma_restrack_parent_name(&mr->res, &pd->res);
2157 	rdma_restrack_add(&mr->res);
2158 
2159 	return mr;
2160 }
2161 EXPORT_SYMBOL(ib_reg_user_mr);
2162 
ib_advise_mr(struct ib_pd * pd,enum ib_uverbs_advise_mr_advice advice,u32 flags,struct ib_sge * sg_list,u32 num_sge)2163 int ib_advise_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
2164 		 u32 flags, struct ib_sge *sg_list, u32 num_sge)
2165 {
2166 	if (!pd->device->ops.advise_mr)
2167 		return -EOPNOTSUPP;
2168 
2169 	if (!num_sge)
2170 		return 0;
2171 
2172 	return pd->device->ops.advise_mr(pd, advice, flags, sg_list, num_sge,
2173 					 NULL);
2174 }
2175 EXPORT_SYMBOL(ib_advise_mr);
2176 
ib_dereg_mr_user(struct ib_mr * mr,struct ib_udata * udata)2177 int ib_dereg_mr_user(struct ib_mr *mr, struct ib_udata *udata)
2178 {
2179 	struct ib_pd *pd = mr->pd;
2180 	struct ib_dm *dm = mr->dm;
2181 	struct ib_sig_attrs *sig_attrs = mr->sig_attrs;
2182 	int ret;
2183 
2184 	trace_mr_dereg(mr);
2185 	rdma_restrack_del(&mr->res);
2186 	ret = mr->device->ops.dereg_mr(mr, udata);
2187 	if (!ret) {
2188 		atomic_dec(&pd->usecnt);
2189 		if (dm)
2190 			atomic_dec(&dm->usecnt);
2191 		kfree(sig_attrs);
2192 	}
2193 
2194 	return ret;
2195 }
2196 EXPORT_SYMBOL(ib_dereg_mr_user);
2197 
2198 /**
2199  * ib_alloc_mr() - Allocates a memory region
2200  * @pd:            protection domain associated with the region
2201  * @mr_type:       memory region type
2202  * @max_num_sg:    maximum sg entries available for registration.
2203  *
2204  * Notes:
2205  * Memory registeration page/sg lists must not exceed max_num_sg.
2206  * For mr_type IB_MR_TYPE_MEM_REG, the total length cannot exceed
2207  * max_num_sg * used_page_size.
2208  *
2209  */
ib_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_num_sg)2210 struct ib_mr *ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
2211 			  u32 max_num_sg)
2212 {
2213 	struct ib_mr *mr;
2214 
2215 	if (!pd->device->ops.alloc_mr) {
2216 		mr = ERR_PTR(-EOPNOTSUPP);
2217 		goto out;
2218 	}
2219 
2220 	if (mr_type == IB_MR_TYPE_INTEGRITY) {
2221 		WARN_ON_ONCE(1);
2222 		mr = ERR_PTR(-EINVAL);
2223 		goto out;
2224 	}
2225 
2226 	mr = pd->device->ops.alloc_mr(pd, mr_type, max_num_sg);
2227 	if (IS_ERR(mr))
2228 		goto out;
2229 
2230 	mr->device = pd->device;
2231 	mr->pd = pd;
2232 	mr->dm = NULL;
2233 	mr->uobject = NULL;
2234 	atomic_inc(&pd->usecnt);
2235 	mr->need_inval = false;
2236 	mr->type = mr_type;
2237 	mr->sig_attrs = NULL;
2238 
2239 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2240 	rdma_restrack_parent_name(&mr->res, &pd->res);
2241 	rdma_restrack_add(&mr->res);
2242 out:
2243 	trace_mr_alloc(pd, mr_type, max_num_sg, mr);
2244 	return mr;
2245 }
2246 EXPORT_SYMBOL(ib_alloc_mr);
2247 
2248 /**
2249  * ib_alloc_mr_integrity() - Allocates an integrity memory region
2250  * @pd:                      protection domain associated with the region
2251  * @max_num_data_sg:         maximum data sg entries available for registration
2252  * @max_num_meta_sg:         maximum metadata sg entries available for
2253  *                           registration
2254  *
2255  * Notes:
2256  * Memory registration page/sg lists must not exceed max_num_sg,
2257  * also the integrity page/sg lists must not exceed max_num_meta_sg.
2258  *
2259  */
ib_alloc_mr_integrity(struct ib_pd * pd,u32 max_num_data_sg,u32 max_num_meta_sg)2260 struct ib_mr *ib_alloc_mr_integrity(struct ib_pd *pd,
2261 				    u32 max_num_data_sg,
2262 				    u32 max_num_meta_sg)
2263 {
2264 	struct ib_mr *mr;
2265 	struct ib_sig_attrs *sig_attrs;
2266 
2267 	if (!pd->device->ops.alloc_mr_integrity ||
2268 	    !pd->device->ops.map_mr_sg_pi) {
2269 		mr = ERR_PTR(-EOPNOTSUPP);
2270 		goto out;
2271 	}
2272 
2273 	if (!max_num_meta_sg) {
2274 		mr = ERR_PTR(-EINVAL);
2275 		goto out;
2276 	}
2277 
2278 	sig_attrs = kzalloc(sizeof(struct ib_sig_attrs), GFP_KERNEL);
2279 	if (!sig_attrs) {
2280 		mr = ERR_PTR(-ENOMEM);
2281 		goto out;
2282 	}
2283 
2284 	mr = pd->device->ops.alloc_mr_integrity(pd, max_num_data_sg,
2285 						max_num_meta_sg);
2286 	if (IS_ERR(mr)) {
2287 		kfree(sig_attrs);
2288 		goto out;
2289 	}
2290 
2291 	mr->device = pd->device;
2292 	mr->pd = pd;
2293 	mr->dm = NULL;
2294 	mr->uobject = NULL;
2295 	atomic_inc(&pd->usecnt);
2296 	mr->need_inval = false;
2297 	mr->type = IB_MR_TYPE_INTEGRITY;
2298 	mr->sig_attrs = sig_attrs;
2299 
2300 	rdma_restrack_new(&mr->res, RDMA_RESTRACK_MR);
2301 	rdma_restrack_parent_name(&mr->res, &pd->res);
2302 	rdma_restrack_add(&mr->res);
2303 out:
2304 	trace_mr_integ_alloc(pd, max_num_data_sg, max_num_meta_sg, mr);
2305 	return mr;
2306 }
2307 EXPORT_SYMBOL(ib_alloc_mr_integrity);
2308 
2309 /* Multicast groups */
2310 
is_valid_mcast_lid(struct ib_qp * qp,u16 lid)2311 static bool is_valid_mcast_lid(struct ib_qp *qp, u16 lid)
2312 {
2313 	struct ib_qp_init_attr init_attr = {};
2314 	struct ib_qp_attr attr = {};
2315 	int num_eth_ports = 0;
2316 	unsigned int port;
2317 
2318 	/* If QP state >= init, it is assigned to a port and we can check this
2319 	 * port only.
2320 	 */
2321 	if (!ib_query_qp(qp, &attr, IB_QP_STATE | IB_QP_PORT, &init_attr)) {
2322 		if (attr.qp_state >= IB_QPS_INIT) {
2323 			if (rdma_port_get_link_layer(qp->device, attr.port_num) !=
2324 			    IB_LINK_LAYER_INFINIBAND)
2325 				return true;
2326 			goto lid_check;
2327 		}
2328 	}
2329 
2330 	/* Can't get a quick answer, iterate over all ports */
2331 	rdma_for_each_port(qp->device, port)
2332 		if (rdma_port_get_link_layer(qp->device, port) !=
2333 		    IB_LINK_LAYER_INFINIBAND)
2334 			num_eth_ports++;
2335 
2336 	/* If we have at lease one Ethernet port, RoCE annex declares that
2337 	 * multicast LID should be ignored. We can't tell at this step if the
2338 	 * QP belongs to an IB or Ethernet port.
2339 	 */
2340 	if (num_eth_ports)
2341 		return true;
2342 
2343 	/* If all the ports are IB, we can check according to IB spec. */
2344 lid_check:
2345 	return !(lid < be16_to_cpu(IB_MULTICAST_LID_BASE) ||
2346 		 lid == be16_to_cpu(IB_LID_PERMISSIVE));
2347 }
2348 
ib_attach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)2349 int ib_attach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2350 {
2351 	int ret;
2352 
2353 	if (!qp->device->ops.attach_mcast)
2354 		return -EOPNOTSUPP;
2355 
2356 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2357 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2358 		return -EINVAL;
2359 
2360 	ret = qp->device->ops.attach_mcast(qp, gid, lid);
2361 	if (!ret)
2362 		atomic_inc(&qp->usecnt);
2363 	return ret;
2364 }
2365 EXPORT_SYMBOL(ib_attach_mcast);
2366 
ib_detach_mcast(struct ib_qp * qp,union ib_gid * gid,u16 lid)2367 int ib_detach_mcast(struct ib_qp *qp, union ib_gid *gid, u16 lid)
2368 {
2369 	int ret;
2370 
2371 	if (!qp->device->ops.detach_mcast)
2372 		return -EOPNOTSUPP;
2373 
2374 	if (!rdma_is_multicast_addr((struct in6_addr *)gid->raw) ||
2375 	    qp->qp_type != IB_QPT_UD || !is_valid_mcast_lid(qp, lid))
2376 		return -EINVAL;
2377 
2378 	ret = qp->device->ops.detach_mcast(qp, gid, lid);
2379 	if (!ret)
2380 		atomic_dec(&qp->usecnt);
2381 	return ret;
2382 }
2383 EXPORT_SYMBOL(ib_detach_mcast);
2384 
2385 /**
2386  * ib_alloc_xrcd_user - Allocates an XRC domain.
2387  * @device: The device on which to allocate the XRC domain.
2388  * @inode: inode to connect XRCD
2389  * @udata: Valid user data or NULL for kernel object
2390  */
ib_alloc_xrcd_user(struct ib_device * device,struct inode * inode,struct ib_udata * udata)2391 struct ib_xrcd *ib_alloc_xrcd_user(struct ib_device *device,
2392 				   struct inode *inode, struct ib_udata *udata)
2393 {
2394 	struct ib_xrcd *xrcd;
2395 	int ret;
2396 
2397 	if (!device->ops.alloc_xrcd)
2398 		return ERR_PTR(-EOPNOTSUPP);
2399 
2400 	xrcd = rdma_zalloc_drv_obj(device, ib_xrcd);
2401 	if (!xrcd)
2402 		return ERR_PTR(-ENOMEM);
2403 
2404 	xrcd->device = device;
2405 	xrcd->inode = inode;
2406 	atomic_set(&xrcd->usecnt, 0);
2407 	init_rwsem(&xrcd->tgt_qps_rwsem);
2408 	xa_init(&xrcd->tgt_qps);
2409 
2410 	ret = device->ops.alloc_xrcd(xrcd, udata);
2411 	if (ret)
2412 		goto err;
2413 	return xrcd;
2414 err:
2415 	kfree(xrcd);
2416 	return ERR_PTR(ret);
2417 }
2418 EXPORT_SYMBOL(ib_alloc_xrcd_user);
2419 
2420 /**
2421  * ib_dealloc_xrcd_user - Deallocates an XRC domain.
2422  * @xrcd: The XRC domain to deallocate.
2423  * @udata: Valid user data or NULL for kernel object
2424  */
ib_dealloc_xrcd_user(struct ib_xrcd * xrcd,struct ib_udata * udata)2425 int ib_dealloc_xrcd_user(struct ib_xrcd *xrcd, struct ib_udata *udata)
2426 {
2427 	int ret;
2428 
2429 	if (atomic_read(&xrcd->usecnt))
2430 		return -EBUSY;
2431 
2432 	WARN_ON(!xa_empty(&xrcd->tgt_qps));
2433 	ret = xrcd->device->ops.dealloc_xrcd(xrcd, udata);
2434 	if (ret)
2435 		return ret;
2436 	kfree(xrcd);
2437 	return ret;
2438 }
2439 EXPORT_SYMBOL(ib_dealloc_xrcd_user);
2440 
2441 /**
2442  * ib_create_wq - Creates a WQ associated with the specified protection
2443  * domain.
2444  * @pd: The protection domain associated with the WQ.
2445  * @wq_attr: A list of initial attributes required to create the
2446  * WQ. If WQ creation succeeds, then the attributes are updated to
2447  * the actual capabilities of the created WQ.
2448  *
2449  * wq_attr->max_wr and wq_attr->max_sge determine
2450  * the requested size of the WQ, and set to the actual values allocated
2451  * on return.
2452  * If ib_create_wq() succeeds, then max_wr and max_sge will always be
2453  * at least as large as the requested values.
2454  */
ib_create_wq(struct ib_pd * pd,struct ib_wq_init_attr * wq_attr)2455 struct ib_wq *ib_create_wq(struct ib_pd *pd,
2456 			   struct ib_wq_init_attr *wq_attr)
2457 {
2458 	struct ib_wq *wq;
2459 
2460 	if (!pd->device->ops.create_wq)
2461 		return ERR_PTR(-EOPNOTSUPP);
2462 
2463 	wq = pd->device->ops.create_wq(pd, wq_attr, NULL);
2464 	if (!IS_ERR(wq)) {
2465 		wq->event_handler = wq_attr->event_handler;
2466 		wq->wq_context = wq_attr->wq_context;
2467 		wq->wq_type = wq_attr->wq_type;
2468 		wq->cq = wq_attr->cq;
2469 		wq->device = pd->device;
2470 		wq->pd = pd;
2471 		wq->uobject = NULL;
2472 		atomic_inc(&pd->usecnt);
2473 		atomic_inc(&wq_attr->cq->usecnt);
2474 		atomic_set(&wq->usecnt, 0);
2475 	}
2476 	return wq;
2477 }
2478 EXPORT_SYMBOL(ib_create_wq);
2479 
2480 /**
2481  * ib_destroy_wq_user - Destroys the specified user WQ.
2482  * @wq: The WQ to destroy.
2483  * @udata: Valid user data
2484  */
ib_destroy_wq_user(struct ib_wq * wq,struct ib_udata * udata)2485 int ib_destroy_wq_user(struct ib_wq *wq, struct ib_udata *udata)
2486 {
2487 	struct ib_cq *cq = wq->cq;
2488 	struct ib_pd *pd = wq->pd;
2489 	int ret;
2490 
2491 	if (atomic_read(&wq->usecnt))
2492 		return -EBUSY;
2493 
2494 	ret = wq->device->ops.destroy_wq(wq, udata);
2495 	if (ret)
2496 		return ret;
2497 
2498 	atomic_dec(&pd->usecnt);
2499 	atomic_dec(&cq->usecnt);
2500 	return ret;
2501 }
2502 EXPORT_SYMBOL(ib_destroy_wq_user);
2503 
ib_check_mr_status(struct ib_mr * mr,u32 check_mask,struct ib_mr_status * mr_status)2504 int ib_check_mr_status(struct ib_mr *mr, u32 check_mask,
2505 		       struct ib_mr_status *mr_status)
2506 {
2507 	if (!mr->device->ops.check_mr_status)
2508 		return -EOPNOTSUPP;
2509 
2510 	return mr->device->ops.check_mr_status(mr, check_mask, mr_status);
2511 }
2512 EXPORT_SYMBOL(ib_check_mr_status);
2513 
ib_set_vf_link_state(struct ib_device * device,int vf,u32 port,int state)2514 int ib_set_vf_link_state(struct ib_device *device, int vf, u32 port,
2515 			 int state)
2516 {
2517 	if (!device->ops.set_vf_link_state)
2518 		return -EOPNOTSUPP;
2519 
2520 	return device->ops.set_vf_link_state(device, vf, port, state);
2521 }
2522 EXPORT_SYMBOL(ib_set_vf_link_state);
2523 
ib_get_vf_config(struct ib_device * device,int vf,u32 port,struct ifla_vf_info * info)2524 int ib_get_vf_config(struct ib_device *device, int vf, u32 port,
2525 		     struct ifla_vf_info *info)
2526 {
2527 	if (!device->ops.get_vf_config)
2528 		return -EOPNOTSUPP;
2529 
2530 	return device->ops.get_vf_config(device, vf, port, info);
2531 }
2532 EXPORT_SYMBOL(ib_get_vf_config);
2533 
ib_get_vf_stats(struct ib_device * device,int vf,u32 port,struct ifla_vf_stats * stats)2534 int ib_get_vf_stats(struct ib_device *device, int vf, u32 port,
2535 		    struct ifla_vf_stats *stats)
2536 {
2537 	if (!device->ops.get_vf_stats)
2538 		return -EOPNOTSUPP;
2539 
2540 	return device->ops.get_vf_stats(device, vf, port, stats);
2541 }
2542 EXPORT_SYMBOL(ib_get_vf_stats);
2543 
ib_set_vf_guid(struct ib_device * device,int vf,u32 port,u64 guid,int type)2544 int ib_set_vf_guid(struct ib_device *device, int vf, u32 port, u64 guid,
2545 		   int type)
2546 {
2547 	if (!device->ops.set_vf_guid)
2548 		return -EOPNOTSUPP;
2549 
2550 	return device->ops.set_vf_guid(device, vf, port, guid, type);
2551 }
2552 EXPORT_SYMBOL(ib_set_vf_guid);
2553 
ib_get_vf_guid(struct ib_device * device,int vf,u32 port,struct ifla_vf_guid * node_guid,struct ifla_vf_guid * port_guid)2554 int ib_get_vf_guid(struct ib_device *device, int vf, u32 port,
2555 		   struct ifla_vf_guid *node_guid,
2556 		   struct ifla_vf_guid *port_guid)
2557 {
2558 	if (!device->ops.get_vf_guid)
2559 		return -EOPNOTSUPP;
2560 
2561 	return device->ops.get_vf_guid(device, vf, port, node_guid, port_guid);
2562 }
2563 EXPORT_SYMBOL(ib_get_vf_guid);
2564 /**
2565  * ib_map_mr_sg_pi() - Map the dma mapped SG lists for PI (protection
2566  *     information) and set an appropriate memory region for registration.
2567  * @mr:             memory region
2568  * @data_sg:        dma mapped scatterlist for data
2569  * @data_sg_nents:  number of entries in data_sg
2570  * @data_sg_offset: offset in bytes into data_sg
2571  * @meta_sg:        dma mapped scatterlist for metadata
2572  * @meta_sg_nents:  number of entries in meta_sg
2573  * @meta_sg_offset: offset in bytes into meta_sg
2574  * @page_size:      page vector desired page size
2575  *
2576  * Constraints:
2577  * - The MR must be allocated with type IB_MR_TYPE_INTEGRITY.
2578  *
2579  * Return: 0 on success.
2580  *
2581  * After this completes successfully, the  memory region
2582  * is ready for registration.
2583  */
ib_map_mr_sg_pi(struct ib_mr * mr,struct scatterlist * data_sg,int data_sg_nents,unsigned int * data_sg_offset,struct scatterlist * meta_sg,int meta_sg_nents,unsigned int * meta_sg_offset,unsigned int page_size)2584 int ib_map_mr_sg_pi(struct ib_mr *mr, struct scatterlist *data_sg,
2585 		    int data_sg_nents, unsigned int *data_sg_offset,
2586 		    struct scatterlist *meta_sg, int meta_sg_nents,
2587 		    unsigned int *meta_sg_offset, unsigned int page_size)
2588 {
2589 	if (unlikely(!mr->device->ops.map_mr_sg_pi ||
2590 		     WARN_ON_ONCE(mr->type != IB_MR_TYPE_INTEGRITY)))
2591 		return -EOPNOTSUPP;
2592 
2593 	mr->page_size = page_size;
2594 
2595 	return mr->device->ops.map_mr_sg_pi(mr, data_sg, data_sg_nents,
2596 					    data_sg_offset, meta_sg,
2597 					    meta_sg_nents, meta_sg_offset);
2598 }
2599 EXPORT_SYMBOL(ib_map_mr_sg_pi);
2600 
2601 /**
2602  * ib_map_mr_sg() - Map the largest prefix of a dma mapped SG list
2603  *     and set it the memory region.
2604  * @mr:            memory region
2605  * @sg:            dma mapped scatterlist
2606  * @sg_nents:      number of entries in sg
2607  * @sg_offset:     offset in bytes into sg
2608  * @page_size:     page vector desired page size
2609  *
2610  * Constraints:
2611  *
2612  * - The first sg element is allowed to have an offset.
2613  * - Each sg element must either be aligned to page_size or virtually
2614  *   contiguous to the previous element. In case an sg element has a
2615  *   non-contiguous offset, the mapping prefix will not include it.
2616  * - The last sg element is allowed to have length less than page_size.
2617  * - If sg_nents total byte length exceeds the mr max_num_sge * page_size
2618  *   then only max_num_sg entries will be mapped.
2619  * - If the MR was allocated with type IB_MR_TYPE_SG_GAPS, none of these
2620  *   constraints holds and the page_size argument is ignored.
2621  *
2622  * Returns the number of sg elements that were mapped to the memory region.
2623  *
2624  * After this completes successfully, the  memory region
2625  * is ready for registration.
2626  */
ib_map_mr_sg(struct ib_mr * mr,struct scatterlist * sg,int sg_nents,unsigned int * sg_offset,unsigned int page_size)2627 int ib_map_mr_sg(struct ib_mr *mr, struct scatterlist *sg, int sg_nents,
2628 		 unsigned int *sg_offset, unsigned int page_size)
2629 {
2630 	if (unlikely(!mr->device->ops.map_mr_sg))
2631 		return -EOPNOTSUPP;
2632 
2633 	mr->page_size = page_size;
2634 
2635 	return mr->device->ops.map_mr_sg(mr, sg, sg_nents, sg_offset);
2636 }
2637 EXPORT_SYMBOL(ib_map_mr_sg);
2638 
2639 /**
2640  * ib_sg_to_pages() - Convert the largest prefix of a sg list
2641  *     to a page vector
2642  * @mr:            memory region
2643  * @sgl:           dma mapped scatterlist
2644  * @sg_nents:      number of entries in sg
2645  * @sg_offset_p:   ==== =======================================================
2646  *                 IN   start offset in bytes into sg
2647  *                 OUT  offset in bytes for element n of the sg of the first
2648  *                      byte that has not been processed where n is the return
2649  *                      value of this function.
2650  *                 ==== =======================================================
2651  * @set_page:      driver page assignment function pointer
2652  *
2653  * Core service helper for drivers to convert the largest
2654  * prefix of given sg list to a page vector. The sg list
2655  * prefix converted is the prefix that meet the requirements
2656  * of ib_map_mr_sg.
2657  *
2658  * Returns the number of sg elements that were assigned to
2659  * a page vector.
2660  */
ib_sg_to_pages(struct ib_mr * mr,struct scatterlist * sgl,int sg_nents,unsigned int * sg_offset_p,int (* set_page)(struct ib_mr *,u64))2661 int ib_sg_to_pages(struct ib_mr *mr, struct scatterlist *sgl, int sg_nents,
2662 		unsigned int *sg_offset_p, int (*set_page)(struct ib_mr *, u64))
2663 {
2664 	struct scatterlist *sg;
2665 	u64 last_end_dma_addr = 0;
2666 	unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0;
2667 	unsigned int last_page_off = 0;
2668 	u64 page_mask = ~((u64)mr->page_size - 1);
2669 	int i, ret;
2670 
2671 	if (unlikely(sg_nents <= 0 || sg_offset > sg_dma_len(&sgl[0])))
2672 		return -EINVAL;
2673 
2674 	mr->iova = sg_dma_address(&sgl[0]) + sg_offset;
2675 	mr->length = 0;
2676 
2677 	for_each_sg(sgl, sg, sg_nents, i) {
2678 		u64 dma_addr = sg_dma_address(sg) + sg_offset;
2679 		u64 prev_addr = dma_addr;
2680 		unsigned int dma_len = sg_dma_len(sg) - sg_offset;
2681 		u64 end_dma_addr = dma_addr + dma_len;
2682 		u64 page_addr = dma_addr & page_mask;
2683 
2684 		/*
2685 		 * For the second and later elements, check whether either the
2686 		 * end of element i-1 or the start of element i is not aligned
2687 		 * on a page boundary.
2688 		 */
2689 		if (i && (last_page_off != 0 || page_addr != dma_addr)) {
2690 			/* Stop mapping if there is a gap. */
2691 			if (last_end_dma_addr != dma_addr)
2692 				break;
2693 
2694 			/*
2695 			 * Coalesce this element with the last. If it is small
2696 			 * enough just update mr->length. Otherwise start
2697 			 * mapping from the next page.
2698 			 */
2699 			goto next_page;
2700 		}
2701 
2702 		do {
2703 			ret = set_page(mr, page_addr);
2704 			if (unlikely(ret < 0)) {
2705 				sg_offset = prev_addr - sg_dma_address(sg);
2706 				mr->length += prev_addr - dma_addr;
2707 				if (sg_offset_p)
2708 					*sg_offset_p = sg_offset;
2709 				return i || sg_offset ? i : ret;
2710 			}
2711 			prev_addr = page_addr;
2712 next_page:
2713 			page_addr += mr->page_size;
2714 		} while (page_addr < end_dma_addr);
2715 
2716 		mr->length += dma_len;
2717 		last_end_dma_addr = end_dma_addr;
2718 		last_page_off = end_dma_addr & ~page_mask;
2719 
2720 		sg_offset = 0;
2721 	}
2722 
2723 	if (sg_offset_p)
2724 		*sg_offset_p = 0;
2725 	return i;
2726 }
2727 EXPORT_SYMBOL(ib_sg_to_pages);
2728 
2729 struct ib_drain_cqe {
2730 	struct ib_cqe cqe;
2731 	struct completion done;
2732 };
2733 
ib_drain_qp_done(struct ib_cq * cq,struct ib_wc * wc)2734 static void ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
2735 {
2736 	struct ib_drain_cqe *cqe = container_of(wc->wr_cqe, struct ib_drain_cqe,
2737 						cqe);
2738 
2739 	complete(&cqe->done);
2740 }
2741 
2742 /*
2743  * Post a WR and block until its completion is reaped for the SQ.
2744  */
__ib_drain_sq(struct ib_qp * qp)2745 static void __ib_drain_sq(struct ib_qp *qp)
2746 {
2747 	struct ib_cq *cq = qp->send_cq;
2748 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2749 	struct ib_drain_cqe sdrain;
2750 	struct ib_rdma_wr swr = {
2751 		.wr = {
2752 			.next = NULL,
2753 			{ .wr_cqe	= &sdrain.cqe, },
2754 			.opcode	= IB_WR_RDMA_WRITE,
2755 		},
2756 	};
2757 	int ret;
2758 
2759 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2760 	if (ret) {
2761 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2762 		return;
2763 	}
2764 
2765 	sdrain.cqe.done = ib_drain_qp_done;
2766 	init_completion(&sdrain.done);
2767 
2768 	ret = ib_post_send(qp, &swr.wr, NULL);
2769 	if (ret) {
2770 		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
2771 		return;
2772 	}
2773 
2774 	if (cq->poll_ctx == IB_POLL_DIRECT)
2775 		while (wait_for_completion_timeout(&sdrain.done, HZ / 10) <= 0)
2776 			ib_process_cq_direct(cq, -1);
2777 	else
2778 		wait_for_completion(&sdrain.done);
2779 }
2780 
2781 /*
2782  * Post a WR and block until its completion is reaped for the RQ.
2783  */
__ib_drain_rq(struct ib_qp * qp)2784 static void __ib_drain_rq(struct ib_qp *qp)
2785 {
2786 	struct ib_cq *cq = qp->recv_cq;
2787 	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
2788 	struct ib_drain_cqe rdrain;
2789 	struct ib_recv_wr rwr = {};
2790 	int ret;
2791 
2792 	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
2793 	if (ret) {
2794 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2795 		return;
2796 	}
2797 
2798 	rwr.wr_cqe = &rdrain.cqe;
2799 	rdrain.cqe.done = ib_drain_qp_done;
2800 	init_completion(&rdrain.done);
2801 
2802 	ret = ib_post_recv(qp, &rwr, NULL);
2803 	if (ret) {
2804 		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
2805 		return;
2806 	}
2807 
2808 	if (cq->poll_ctx == IB_POLL_DIRECT)
2809 		while (wait_for_completion_timeout(&rdrain.done, HZ / 10) <= 0)
2810 			ib_process_cq_direct(cq, -1);
2811 	else
2812 		wait_for_completion(&rdrain.done);
2813 }
2814 
2815 /**
2816  * ib_drain_sq() - Block until all SQ CQEs have been consumed by the
2817  *		   application.
2818  * @qp:            queue pair to drain
2819  *
2820  * If the device has a provider-specific drain function, then
2821  * call that.  Otherwise call the generic drain function
2822  * __ib_drain_sq().
2823  *
2824  * The caller must:
2825  *
2826  * ensure there is room in the CQ and SQ for the drain work request and
2827  * completion.
2828  *
2829  * allocate the CQ using ib_alloc_cq().
2830  *
2831  * ensure that there are no other contexts that are posting WRs concurrently.
2832  * Otherwise the drain is not guaranteed.
2833  */
ib_drain_sq(struct ib_qp * qp)2834 void ib_drain_sq(struct ib_qp *qp)
2835 {
2836 	if (qp->device->ops.drain_sq)
2837 		qp->device->ops.drain_sq(qp);
2838 	else
2839 		__ib_drain_sq(qp);
2840 	trace_cq_drain_complete(qp->send_cq);
2841 }
2842 EXPORT_SYMBOL(ib_drain_sq);
2843 
2844 /**
2845  * ib_drain_rq() - Block until all RQ CQEs have been consumed by the
2846  *		   application.
2847  * @qp:            queue pair to drain
2848  *
2849  * If the device has a provider-specific drain function, then
2850  * call that.  Otherwise call the generic drain function
2851  * __ib_drain_rq().
2852  *
2853  * The caller must:
2854  *
2855  * ensure there is room in the CQ and RQ for the drain work request and
2856  * completion.
2857  *
2858  * allocate the CQ using ib_alloc_cq().
2859  *
2860  * ensure that there are no other contexts that are posting WRs concurrently.
2861  * Otherwise the drain is not guaranteed.
2862  */
ib_drain_rq(struct ib_qp * qp)2863 void ib_drain_rq(struct ib_qp *qp)
2864 {
2865 	if (qp->device->ops.drain_rq)
2866 		qp->device->ops.drain_rq(qp);
2867 	else
2868 		__ib_drain_rq(qp);
2869 	trace_cq_drain_complete(qp->recv_cq);
2870 }
2871 EXPORT_SYMBOL(ib_drain_rq);
2872 
2873 /**
2874  * ib_drain_qp() - Block until all CQEs have been consumed by the
2875  *		   application on both the RQ and SQ.
2876  * @qp:            queue pair to drain
2877  *
2878  * The caller must:
2879  *
2880  * ensure there is room in the CQ(s), SQ, and RQ for drain work requests
2881  * and completions.
2882  *
2883  * allocate the CQs using ib_alloc_cq().
2884  *
2885  * ensure that there are no other contexts that are posting WRs concurrently.
2886  * Otherwise the drain is not guaranteed.
2887  */
ib_drain_qp(struct ib_qp * qp)2888 void ib_drain_qp(struct ib_qp *qp)
2889 {
2890 	ib_drain_sq(qp);
2891 	if (!qp->srq)
2892 		ib_drain_rq(qp);
2893 }
2894 EXPORT_SYMBOL(ib_drain_qp);
2895 
rdma_alloc_netdev(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *))2896 struct net_device *rdma_alloc_netdev(struct ib_device *device, u32 port_num,
2897 				     enum rdma_netdev_t type, const char *name,
2898 				     unsigned char name_assign_type,
2899 				     void (*setup)(struct net_device *))
2900 {
2901 	struct rdma_netdev_alloc_params params;
2902 	struct net_device *netdev;
2903 	int rc;
2904 
2905 	if (!device->ops.rdma_netdev_get_params)
2906 		return ERR_PTR(-EOPNOTSUPP);
2907 
2908 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2909 						&params);
2910 	if (rc)
2911 		return ERR_PTR(rc);
2912 
2913 	netdev = alloc_netdev_mqs(params.sizeof_priv, name, name_assign_type,
2914 				  setup, params.txqs, params.rxqs);
2915 	if (!netdev)
2916 		return ERR_PTR(-ENOMEM);
2917 
2918 	return netdev;
2919 }
2920 EXPORT_SYMBOL(rdma_alloc_netdev);
2921 
rdma_init_netdev(struct ib_device * device,u32 port_num,enum rdma_netdev_t type,const char * name,unsigned char name_assign_type,void (* setup)(struct net_device *),struct net_device * netdev)2922 int rdma_init_netdev(struct ib_device *device, u32 port_num,
2923 		     enum rdma_netdev_t type, const char *name,
2924 		     unsigned char name_assign_type,
2925 		     void (*setup)(struct net_device *),
2926 		     struct net_device *netdev)
2927 {
2928 	struct rdma_netdev_alloc_params params;
2929 	int rc;
2930 
2931 	if (!device->ops.rdma_netdev_get_params)
2932 		return -EOPNOTSUPP;
2933 
2934 	rc = device->ops.rdma_netdev_get_params(device, port_num, type,
2935 						&params);
2936 	if (rc)
2937 		return rc;
2938 
2939 	return params.initialize_rdma_netdev(device, port_num,
2940 					     netdev, params.param);
2941 }
2942 EXPORT_SYMBOL(rdma_init_netdev);
2943 
__rdma_block_iter_start(struct ib_block_iter * biter,struct scatterlist * sglist,unsigned int nents,unsigned long pgsz)2944 void __rdma_block_iter_start(struct ib_block_iter *biter,
2945 			     struct scatterlist *sglist, unsigned int nents,
2946 			     unsigned long pgsz)
2947 {
2948 	memset(biter, 0, sizeof(struct ib_block_iter));
2949 	biter->__sg = sglist;
2950 	biter->__sg_nents = nents;
2951 
2952 	/* Driver provides best block size to use */
2953 	biter->__pg_bit = __fls(pgsz);
2954 }
2955 EXPORT_SYMBOL(__rdma_block_iter_start);
2956 
__rdma_block_iter_next(struct ib_block_iter * biter)2957 bool __rdma_block_iter_next(struct ib_block_iter *biter)
2958 {
2959 	unsigned int block_offset;
2960 	unsigned int sg_delta;
2961 
2962 	if (!biter->__sg_nents || !biter->__sg)
2963 		return false;
2964 
2965 	biter->__dma_addr = sg_dma_address(biter->__sg) + biter->__sg_advance;
2966 	block_offset = biter->__dma_addr & (BIT_ULL(biter->__pg_bit) - 1);
2967 	sg_delta = BIT_ULL(biter->__pg_bit) - block_offset;
2968 
2969 	if (sg_dma_len(biter->__sg) - biter->__sg_advance > sg_delta) {
2970 		biter->__sg_advance += sg_delta;
2971 	} else {
2972 		biter->__sg_advance = 0;
2973 		biter->__sg = sg_next(biter->__sg);
2974 		biter->__sg_nents--;
2975 	}
2976 
2977 	return true;
2978 }
2979 EXPORT_SYMBOL(__rdma_block_iter_next);
2980 
2981 /**
2982  * rdma_alloc_hw_stats_struct - Helper function to allocate dynamic struct
2983  *   for the drivers.
2984  * @descs: array of static descriptors
2985  * @num_counters: number of elements in array
2986  * @lifespan: milliseconds between updates
2987  */
rdma_alloc_hw_stats_struct(const struct rdma_stat_desc * descs,int num_counters,unsigned long lifespan)2988 struct rdma_hw_stats *rdma_alloc_hw_stats_struct(
2989 	const struct rdma_stat_desc *descs, int num_counters,
2990 	unsigned long lifespan)
2991 {
2992 	struct rdma_hw_stats *stats;
2993 
2994 	stats = kzalloc(struct_size(stats, value, num_counters), GFP_KERNEL);
2995 	if (!stats)
2996 		return NULL;
2997 
2998 	stats->is_disabled = kcalloc(BITS_TO_LONGS(num_counters),
2999 				     sizeof(*stats->is_disabled), GFP_KERNEL);
3000 	if (!stats->is_disabled)
3001 		goto err;
3002 
3003 	stats->descs = descs;
3004 	stats->num_counters = num_counters;
3005 	stats->lifespan = msecs_to_jiffies(lifespan);
3006 	mutex_init(&stats->lock);
3007 
3008 	return stats;
3009 
3010 err:
3011 	kfree(stats);
3012 	return NULL;
3013 }
3014 EXPORT_SYMBOL(rdma_alloc_hw_stats_struct);
3015 
3016 /**
3017  * rdma_free_hw_stats_struct - Helper function to release rdma_hw_stats
3018  * @stats: statistics to release
3019  */
rdma_free_hw_stats_struct(struct rdma_hw_stats * stats)3020 void rdma_free_hw_stats_struct(struct rdma_hw_stats *stats)
3021 {
3022 	if (!stats)
3023 		return;
3024 
3025 	kfree(stats->is_disabled);
3026 	kfree(stats);
3027 }
3028 EXPORT_SYMBOL(rdma_free_hw_stats_struct);
3029