1 // SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2 
3 /* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4 /* Copyright (c) 2008-2019, IBM Corporation */
5 
6 #include <linux/errno.h>
7 #include <linux/types.h>
8 #include <linux/uaccess.h>
9 #include <linux/vmalloc.h>
10 #include <linux/xarray.h>
11 #include <net/addrconf.h>
12 
13 #include <rdma/iw_cm.h>
14 #include <rdma/ib_verbs.h>
15 #include <rdma/ib_user_verbs.h>
16 #include <rdma/uverbs_ioctl.h>
17 
18 #include "siw.h"
19 #include "siw_verbs.h"
20 #include "siw_mem.h"
21 
22 static int ib_qp_state_to_siw_qp_state[IB_QPS_ERR + 1] = {
23 	[IB_QPS_RESET] = SIW_QP_STATE_IDLE,
24 	[IB_QPS_INIT] = SIW_QP_STATE_IDLE,
25 	[IB_QPS_RTR] = SIW_QP_STATE_RTR,
26 	[IB_QPS_RTS] = SIW_QP_STATE_RTS,
27 	[IB_QPS_SQD] = SIW_QP_STATE_CLOSING,
28 	[IB_QPS_SQE] = SIW_QP_STATE_TERMINATE,
29 	[IB_QPS_ERR] = SIW_QP_STATE_ERROR
30 };
31 
32 static char ib_qp_state_to_string[IB_QPS_ERR + 1][sizeof("RESET")] = {
33 	[IB_QPS_RESET] = "RESET", [IB_QPS_INIT] = "INIT", [IB_QPS_RTR] = "RTR",
34 	[IB_QPS_RTS] = "RTS",     [IB_QPS_SQD] = "SQD",   [IB_QPS_SQE] = "SQE",
35 	[IB_QPS_ERR] = "ERR"
36 };
37 
siw_mmap_free(struct rdma_user_mmap_entry * rdma_entry)38 void siw_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
39 {
40 	struct siw_user_mmap_entry *entry = to_siw_mmap_entry(rdma_entry);
41 
42 	kfree(entry);
43 }
44 
siw_mmap(struct ib_ucontext * ctx,struct vm_area_struct * vma)45 int siw_mmap(struct ib_ucontext *ctx, struct vm_area_struct *vma)
46 {
47 	struct siw_ucontext *uctx = to_siw_ctx(ctx);
48 	size_t size = vma->vm_end - vma->vm_start;
49 	struct rdma_user_mmap_entry *rdma_entry;
50 	struct siw_user_mmap_entry *entry;
51 	int rv = -EINVAL;
52 
53 	/*
54 	 * Must be page aligned
55 	 */
56 	if (vma->vm_start & (PAGE_SIZE - 1)) {
57 		pr_warn("siw: mmap not page aligned\n");
58 		return -EINVAL;
59 	}
60 	rdma_entry = rdma_user_mmap_entry_get(&uctx->base_ucontext, vma);
61 	if (!rdma_entry) {
62 		siw_dbg(&uctx->sdev->base_dev, "mmap lookup failed: %lu, %#zx\n",
63 			vma->vm_pgoff, size);
64 		return -EINVAL;
65 	}
66 	entry = to_siw_mmap_entry(rdma_entry);
67 
68 	rv = remap_vmalloc_range(vma, entry->address, 0);
69 	if (rv) {
70 		pr_warn("remap_vmalloc_range failed: %lu, %zu\n", vma->vm_pgoff,
71 			size);
72 		goto out;
73 	}
74 out:
75 	rdma_user_mmap_entry_put(rdma_entry);
76 
77 	return rv;
78 }
79 
siw_alloc_ucontext(struct ib_ucontext * base_ctx,struct ib_udata * udata)80 int siw_alloc_ucontext(struct ib_ucontext *base_ctx, struct ib_udata *udata)
81 {
82 	struct siw_device *sdev = to_siw_dev(base_ctx->device);
83 	struct siw_ucontext *ctx = to_siw_ctx(base_ctx);
84 	struct siw_uresp_alloc_ctx uresp = {};
85 	int rv;
86 
87 	if (atomic_inc_return(&sdev->num_ctx) > SIW_MAX_CONTEXT) {
88 		rv = -ENOMEM;
89 		goto err_out;
90 	}
91 	ctx->sdev = sdev;
92 
93 	uresp.dev_id = sdev->vendor_part_id;
94 
95 	if (udata->outlen < sizeof(uresp)) {
96 		rv = -EINVAL;
97 		goto err_out;
98 	}
99 	rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
100 	if (rv)
101 		goto err_out;
102 
103 	siw_dbg(base_ctx->device, "success. now %d context(s)\n",
104 		atomic_read(&sdev->num_ctx));
105 
106 	return 0;
107 
108 err_out:
109 	atomic_dec(&sdev->num_ctx);
110 	siw_dbg(base_ctx->device, "failure %d. now %d context(s)\n", rv,
111 		atomic_read(&sdev->num_ctx));
112 
113 	return rv;
114 }
115 
siw_dealloc_ucontext(struct ib_ucontext * base_ctx)116 void siw_dealloc_ucontext(struct ib_ucontext *base_ctx)
117 {
118 	struct siw_ucontext *uctx = to_siw_ctx(base_ctx);
119 
120 	atomic_dec(&uctx->sdev->num_ctx);
121 }
122 
siw_query_device(struct ib_device * base_dev,struct ib_device_attr * attr,struct ib_udata * udata)123 int siw_query_device(struct ib_device *base_dev, struct ib_device_attr *attr,
124 		     struct ib_udata *udata)
125 {
126 	struct siw_device *sdev = to_siw_dev(base_dev);
127 
128 	if (udata->inlen || udata->outlen)
129 		return -EINVAL;
130 
131 	memset(attr, 0, sizeof(*attr));
132 
133 	/* Revisit atomic caps if RFC 7306 gets supported */
134 	attr->atomic_cap = 0;
135 	attr->device_cap_flags = IB_DEVICE_MEM_MGT_EXTENSIONS;
136 	attr->kernel_cap_flags = IBK_ALLOW_USER_UNREG;
137 	attr->max_cq = sdev->attrs.max_cq;
138 	attr->max_cqe = sdev->attrs.max_cqe;
139 	attr->max_fast_reg_page_list_len = SIW_MAX_SGE_PBL;
140 	attr->max_mr = sdev->attrs.max_mr;
141 	attr->max_mw = sdev->attrs.max_mw;
142 	attr->max_mr_size = ~0ull;
143 	attr->max_pd = sdev->attrs.max_pd;
144 	attr->max_qp = sdev->attrs.max_qp;
145 	attr->max_qp_init_rd_atom = sdev->attrs.max_ird;
146 	attr->max_qp_rd_atom = sdev->attrs.max_ord;
147 	attr->max_qp_wr = sdev->attrs.max_qp_wr;
148 	attr->max_recv_sge = sdev->attrs.max_sge;
149 	attr->max_res_rd_atom = sdev->attrs.max_qp * sdev->attrs.max_ird;
150 	attr->max_send_sge = sdev->attrs.max_sge;
151 	attr->max_sge_rd = sdev->attrs.max_sge_rd;
152 	attr->max_srq = sdev->attrs.max_srq;
153 	attr->max_srq_sge = sdev->attrs.max_srq_sge;
154 	attr->max_srq_wr = sdev->attrs.max_srq_wr;
155 	attr->page_size_cap = PAGE_SIZE;
156 	attr->vendor_id = SIW_VENDOR_ID;
157 	attr->vendor_part_id = sdev->vendor_part_id;
158 
159 	addrconf_addr_eui48((u8 *)&attr->sys_image_guid,
160 			    sdev->netdev->dev_addr);
161 
162 	return 0;
163 }
164 
siw_query_port(struct ib_device * base_dev,u32 port,struct ib_port_attr * attr)165 int siw_query_port(struct ib_device *base_dev, u32 port,
166 		   struct ib_port_attr *attr)
167 {
168 	struct siw_device *sdev = to_siw_dev(base_dev);
169 	int rv;
170 
171 	memset(attr, 0, sizeof(*attr));
172 
173 	rv = ib_get_eth_speed(base_dev, port, &attr->active_speed,
174 			 &attr->active_width);
175 	attr->gid_tbl_len = 1;
176 	attr->max_msg_sz = -1;
177 	attr->max_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
178 	attr->active_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
179 	attr->phys_state = sdev->state == IB_PORT_ACTIVE ?
180 		IB_PORT_PHYS_STATE_LINK_UP : IB_PORT_PHYS_STATE_DISABLED;
181 	attr->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_DEVICE_MGMT_SUP;
182 	attr->state = sdev->state;
183 	/*
184 	 * All zero
185 	 *
186 	 * attr->lid = 0;
187 	 * attr->bad_pkey_cntr = 0;
188 	 * attr->qkey_viol_cntr = 0;
189 	 * attr->sm_lid = 0;
190 	 * attr->lmc = 0;
191 	 * attr->max_vl_num = 0;
192 	 * attr->sm_sl = 0;
193 	 * attr->subnet_timeout = 0;
194 	 * attr->init_type_repy = 0;
195 	 */
196 	return rv;
197 }
198 
siw_get_port_immutable(struct ib_device * base_dev,u32 port,struct ib_port_immutable * port_immutable)199 int siw_get_port_immutable(struct ib_device *base_dev, u32 port,
200 			   struct ib_port_immutable *port_immutable)
201 {
202 	struct ib_port_attr attr;
203 	int rv = siw_query_port(base_dev, port, &attr);
204 
205 	if (rv)
206 		return rv;
207 
208 	port_immutable->gid_tbl_len = attr.gid_tbl_len;
209 	port_immutable->core_cap_flags = RDMA_CORE_PORT_IWARP;
210 
211 	return 0;
212 }
213 
siw_query_gid(struct ib_device * base_dev,u32 port,int idx,union ib_gid * gid)214 int siw_query_gid(struct ib_device *base_dev, u32 port, int idx,
215 		  union ib_gid *gid)
216 {
217 	struct siw_device *sdev = to_siw_dev(base_dev);
218 
219 	/* subnet_prefix == interface_id == 0; */
220 	memset(gid, 0, sizeof(*gid));
221 	memcpy(&gid->raw[0], sdev->netdev->dev_addr, 6);
222 
223 	return 0;
224 }
225 
siw_alloc_pd(struct ib_pd * pd,struct ib_udata * udata)226 int siw_alloc_pd(struct ib_pd *pd, struct ib_udata *udata)
227 {
228 	struct siw_device *sdev = to_siw_dev(pd->device);
229 
230 	if (atomic_inc_return(&sdev->num_pd) > SIW_MAX_PD) {
231 		atomic_dec(&sdev->num_pd);
232 		return -ENOMEM;
233 	}
234 	siw_dbg_pd(pd, "now %d PD's(s)\n", atomic_read(&sdev->num_pd));
235 
236 	return 0;
237 }
238 
siw_dealloc_pd(struct ib_pd * pd,struct ib_udata * udata)239 int siw_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata)
240 {
241 	struct siw_device *sdev = to_siw_dev(pd->device);
242 
243 	siw_dbg_pd(pd, "free PD\n");
244 	atomic_dec(&sdev->num_pd);
245 	return 0;
246 }
247 
siw_qp_get_ref(struct ib_qp * base_qp)248 void siw_qp_get_ref(struct ib_qp *base_qp)
249 {
250 	siw_qp_get(to_siw_qp(base_qp));
251 }
252 
siw_qp_put_ref(struct ib_qp * base_qp)253 void siw_qp_put_ref(struct ib_qp *base_qp)
254 {
255 	siw_qp_put(to_siw_qp(base_qp));
256 }
257 
258 static struct rdma_user_mmap_entry *
siw_mmap_entry_insert(struct siw_ucontext * uctx,void * address,size_t length,u64 * offset)259 siw_mmap_entry_insert(struct siw_ucontext *uctx,
260 		      void *address, size_t length,
261 		      u64 *offset)
262 {
263 	struct siw_user_mmap_entry *entry = kzalloc(sizeof(*entry), GFP_KERNEL);
264 	int rv;
265 
266 	*offset = SIW_INVAL_UOBJ_KEY;
267 	if (!entry)
268 		return NULL;
269 
270 	entry->address = address;
271 
272 	rv = rdma_user_mmap_entry_insert(&uctx->base_ucontext,
273 					 &entry->rdma_entry,
274 					 length);
275 	if (rv) {
276 		kfree(entry);
277 		return NULL;
278 	}
279 
280 	*offset = rdma_user_mmap_get_offset(&entry->rdma_entry);
281 
282 	return &entry->rdma_entry;
283 }
284 
285 /*
286  * siw_create_qp()
287  *
288  * Create QP of requested size on given device.
289  *
290  * @qp:		Queue pait
291  * @attrs:	Initial QP attributes.
292  * @udata:	used to provide QP ID, SQ and RQ size back to user.
293  */
294 
siw_create_qp(struct ib_qp * ibqp,struct ib_qp_init_attr * attrs,struct ib_udata * udata)295 int siw_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
296 		  struct ib_udata *udata)
297 {
298 	struct ib_pd *pd = ibqp->pd;
299 	struct siw_qp *qp = to_siw_qp(ibqp);
300 	struct ib_device *base_dev = pd->device;
301 	struct siw_device *sdev = to_siw_dev(base_dev);
302 	struct siw_ucontext *uctx =
303 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
304 					  base_ucontext);
305 	unsigned long flags;
306 	int num_sqe, num_rqe, rv = 0;
307 	size_t length;
308 
309 	siw_dbg(base_dev, "create new QP\n");
310 
311 	if (attrs->create_flags)
312 		return -EOPNOTSUPP;
313 
314 	if (atomic_inc_return(&sdev->num_qp) > SIW_MAX_QP) {
315 		siw_dbg(base_dev, "too many QP's\n");
316 		rv = -ENOMEM;
317 		goto err_atomic;
318 	}
319 	if (attrs->qp_type != IB_QPT_RC) {
320 		siw_dbg(base_dev, "only RC QP's supported\n");
321 		rv = -EOPNOTSUPP;
322 		goto err_atomic;
323 	}
324 	if ((attrs->cap.max_send_wr > SIW_MAX_QP_WR) ||
325 	    (attrs->cap.max_recv_wr > SIW_MAX_QP_WR) ||
326 	    (attrs->cap.max_send_sge > SIW_MAX_SGE) ||
327 	    (attrs->cap.max_recv_sge > SIW_MAX_SGE)) {
328 		siw_dbg(base_dev, "QP size error\n");
329 		rv = -EINVAL;
330 		goto err_atomic;
331 	}
332 	if (attrs->cap.max_inline_data > SIW_MAX_INLINE) {
333 		siw_dbg(base_dev, "max inline send: %d > %d\n",
334 			attrs->cap.max_inline_data, (int)SIW_MAX_INLINE);
335 		rv = -EINVAL;
336 		goto err_atomic;
337 	}
338 	/*
339 	 * NOTE: we allow for zero element SQ and RQ WQE's SGL's
340 	 * but not for a QP unable to hold any WQE (SQ + RQ)
341 	 */
342 	if (attrs->cap.max_send_wr + attrs->cap.max_recv_wr == 0) {
343 		siw_dbg(base_dev, "QP must have send or receive queue\n");
344 		rv = -EINVAL;
345 		goto err_atomic;
346 	}
347 
348 	if (!attrs->send_cq || (!attrs->recv_cq && !attrs->srq)) {
349 		siw_dbg(base_dev, "send CQ or receive CQ invalid\n");
350 		rv = -EINVAL;
351 		goto err_atomic;
352 	}
353 
354 	init_rwsem(&qp->state_lock);
355 	spin_lock_init(&qp->sq_lock);
356 	spin_lock_init(&qp->rq_lock);
357 	spin_lock_init(&qp->orq_lock);
358 
359 	rv = siw_qp_add(sdev, qp);
360 	if (rv)
361 		goto err_atomic;
362 
363 	num_sqe = attrs->cap.max_send_wr;
364 	num_rqe = attrs->cap.max_recv_wr;
365 
366 	/* All queue indices are derived from modulo operations
367 	 * on a free running 'get' (consumer) and 'put' (producer)
368 	 * unsigned counter. Having queue sizes at power of two
369 	 * avoids handling counter wrap around.
370 	 */
371 	if (num_sqe)
372 		num_sqe = roundup_pow_of_two(num_sqe);
373 	else {
374 		/* Zero sized SQ is not supported */
375 		rv = -EINVAL;
376 		goto err_out_xa;
377 	}
378 	if (num_rqe)
379 		num_rqe = roundup_pow_of_two(num_rqe);
380 
381 	if (udata)
382 		qp->sendq = vmalloc_user(num_sqe * sizeof(struct siw_sqe));
383 	else
384 		qp->sendq = vzalloc(num_sqe * sizeof(struct siw_sqe));
385 
386 	if (qp->sendq == NULL) {
387 		rv = -ENOMEM;
388 		goto err_out_xa;
389 	}
390 	if (attrs->sq_sig_type != IB_SIGNAL_REQ_WR) {
391 		if (attrs->sq_sig_type == IB_SIGNAL_ALL_WR)
392 			qp->attrs.flags |= SIW_SIGNAL_ALL_WR;
393 		else {
394 			rv = -EINVAL;
395 			goto err_out_xa;
396 		}
397 	}
398 	qp->pd = pd;
399 	qp->scq = to_siw_cq(attrs->send_cq);
400 	qp->rcq = to_siw_cq(attrs->recv_cq);
401 
402 	if (attrs->srq) {
403 		/*
404 		 * SRQ support.
405 		 * Verbs 6.3.7: ignore RQ size, if SRQ present
406 		 * Verbs 6.3.5: do not check PD of SRQ against PD of QP
407 		 */
408 		qp->srq = to_siw_srq(attrs->srq);
409 		qp->attrs.rq_size = 0;
410 		siw_dbg(base_dev, "QP [%u]: SRQ attached\n",
411 			qp->base_qp.qp_num);
412 	} else if (num_rqe) {
413 		if (udata)
414 			qp->recvq =
415 				vmalloc_user(num_rqe * sizeof(struct siw_rqe));
416 		else
417 			qp->recvq = vzalloc(num_rqe * sizeof(struct siw_rqe));
418 
419 		if (qp->recvq == NULL) {
420 			rv = -ENOMEM;
421 			goto err_out_xa;
422 		}
423 		qp->attrs.rq_size = num_rqe;
424 	}
425 	qp->attrs.sq_size = num_sqe;
426 	qp->attrs.sq_max_sges = attrs->cap.max_send_sge;
427 	qp->attrs.rq_max_sges = attrs->cap.max_recv_sge;
428 
429 	/* Make those two tunables fixed for now. */
430 	qp->tx_ctx.gso_seg_limit = 1;
431 	qp->tx_ctx.zcopy_tx = zcopy_tx;
432 
433 	qp->attrs.state = SIW_QP_STATE_IDLE;
434 
435 	if (udata) {
436 		struct siw_uresp_create_qp uresp = {};
437 
438 		uresp.num_sqe = num_sqe;
439 		uresp.num_rqe = num_rqe;
440 		uresp.qp_id = qp_id(qp);
441 
442 		if (qp->sendq) {
443 			length = num_sqe * sizeof(struct siw_sqe);
444 			qp->sq_entry =
445 				siw_mmap_entry_insert(uctx, qp->sendq,
446 						      length, &uresp.sq_key);
447 			if (!qp->sq_entry) {
448 				rv = -ENOMEM;
449 				goto err_out_xa;
450 			}
451 		}
452 
453 		if (qp->recvq) {
454 			length = num_rqe * sizeof(struct siw_rqe);
455 			qp->rq_entry =
456 				siw_mmap_entry_insert(uctx, qp->recvq,
457 						      length, &uresp.rq_key);
458 			if (!qp->rq_entry) {
459 				uresp.sq_key = SIW_INVAL_UOBJ_KEY;
460 				rv = -ENOMEM;
461 				goto err_out_xa;
462 			}
463 		}
464 
465 		if (udata->outlen < sizeof(uresp)) {
466 			rv = -EINVAL;
467 			goto err_out_xa;
468 		}
469 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
470 		if (rv)
471 			goto err_out_xa;
472 	}
473 	qp->tx_cpu = siw_get_tx_cpu(sdev);
474 	if (qp->tx_cpu < 0) {
475 		rv = -EINVAL;
476 		goto err_out_xa;
477 	}
478 	INIT_LIST_HEAD(&qp->devq);
479 	spin_lock_irqsave(&sdev->lock, flags);
480 	list_add_tail(&qp->devq, &sdev->qp_list);
481 	spin_unlock_irqrestore(&sdev->lock, flags);
482 
483 	init_completion(&qp->qp_free);
484 
485 	return 0;
486 
487 err_out_xa:
488 	xa_erase(&sdev->qp_xa, qp_id(qp));
489 	if (uctx) {
490 		rdma_user_mmap_entry_remove(qp->sq_entry);
491 		rdma_user_mmap_entry_remove(qp->rq_entry);
492 	}
493 	vfree(qp->sendq);
494 	vfree(qp->recvq);
495 
496 err_atomic:
497 	atomic_dec(&sdev->num_qp);
498 	return rv;
499 }
500 
501 /*
502  * Minimum siw_query_qp() verb interface.
503  *
504  * @qp_attr_mask is not used but all available information is provided
505  */
siw_query_qp(struct ib_qp * base_qp,struct ib_qp_attr * qp_attr,int qp_attr_mask,struct ib_qp_init_attr * qp_init_attr)506 int siw_query_qp(struct ib_qp *base_qp, struct ib_qp_attr *qp_attr,
507 		 int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
508 {
509 	struct siw_qp *qp;
510 	struct siw_device *sdev;
511 
512 	if (base_qp && qp_attr && qp_init_attr) {
513 		qp = to_siw_qp(base_qp);
514 		sdev = to_siw_dev(base_qp->device);
515 	} else {
516 		return -EINVAL;
517 	}
518 	qp_attr->cap.max_inline_data = SIW_MAX_INLINE;
519 	qp_attr->cap.max_send_wr = qp->attrs.sq_size;
520 	qp_attr->cap.max_send_sge = qp->attrs.sq_max_sges;
521 	qp_attr->cap.max_recv_wr = qp->attrs.rq_size;
522 	qp_attr->cap.max_recv_sge = qp->attrs.rq_max_sges;
523 	qp_attr->path_mtu = ib_mtu_int_to_enum(sdev->netdev->mtu);
524 	qp_attr->max_rd_atomic = qp->attrs.irq_size;
525 	qp_attr->max_dest_rd_atomic = qp->attrs.orq_size;
526 
527 	qp_attr->qp_access_flags = IB_ACCESS_LOCAL_WRITE |
528 				   IB_ACCESS_REMOTE_WRITE |
529 				   IB_ACCESS_REMOTE_READ;
530 
531 	qp_init_attr->qp_type = base_qp->qp_type;
532 	qp_init_attr->send_cq = base_qp->send_cq;
533 	qp_init_attr->recv_cq = base_qp->recv_cq;
534 	qp_init_attr->srq = base_qp->srq;
535 
536 	qp_init_attr->cap = qp_attr->cap;
537 
538 	return 0;
539 }
540 
siw_verbs_modify_qp(struct ib_qp * base_qp,struct ib_qp_attr * attr,int attr_mask,struct ib_udata * udata)541 int siw_verbs_modify_qp(struct ib_qp *base_qp, struct ib_qp_attr *attr,
542 			int attr_mask, struct ib_udata *udata)
543 {
544 	struct siw_qp_attrs new_attrs;
545 	enum siw_qp_attr_mask siw_attr_mask = 0;
546 	struct siw_qp *qp = to_siw_qp(base_qp);
547 	int rv = 0;
548 
549 	if (!attr_mask)
550 		return 0;
551 
552 	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
553 		return -EOPNOTSUPP;
554 
555 	memset(&new_attrs, 0, sizeof(new_attrs));
556 
557 	if (attr_mask & IB_QP_ACCESS_FLAGS) {
558 		siw_attr_mask = SIW_QP_ATTR_ACCESS_FLAGS;
559 
560 		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
561 			new_attrs.flags |= SIW_RDMA_READ_ENABLED;
562 		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
563 			new_attrs.flags |= SIW_RDMA_WRITE_ENABLED;
564 		if (attr->qp_access_flags & IB_ACCESS_MW_BIND)
565 			new_attrs.flags |= SIW_RDMA_BIND_ENABLED;
566 	}
567 	if (attr_mask & IB_QP_STATE) {
568 		siw_dbg_qp(qp, "desired IB QP state: %s\n",
569 			   ib_qp_state_to_string[attr->qp_state]);
570 
571 		new_attrs.state = ib_qp_state_to_siw_qp_state[attr->qp_state];
572 
573 		if (new_attrs.state > SIW_QP_STATE_RTS)
574 			qp->tx_ctx.tx_suspend = 1;
575 
576 		siw_attr_mask |= SIW_QP_ATTR_STATE;
577 	}
578 	if (!siw_attr_mask)
579 		goto out;
580 
581 	down_write(&qp->state_lock);
582 
583 	rv = siw_qp_modify(qp, &new_attrs, siw_attr_mask);
584 
585 	up_write(&qp->state_lock);
586 out:
587 	return rv;
588 }
589 
siw_destroy_qp(struct ib_qp * base_qp,struct ib_udata * udata)590 int siw_destroy_qp(struct ib_qp *base_qp, struct ib_udata *udata)
591 {
592 	struct siw_qp *qp = to_siw_qp(base_qp);
593 	struct siw_ucontext *uctx =
594 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
595 					  base_ucontext);
596 	struct siw_qp_attrs qp_attrs;
597 
598 	siw_dbg_qp(qp, "state %d\n", qp->attrs.state);
599 
600 	/*
601 	 * Mark QP as in process of destruction to prevent from
602 	 * any async callbacks to RDMA core
603 	 */
604 	qp->attrs.flags |= SIW_QP_IN_DESTROY;
605 	qp->rx_stream.rx_suspend = 1;
606 
607 	if (uctx) {
608 		rdma_user_mmap_entry_remove(qp->sq_entry);
609 		rdma_user_mmap_entry_remove(qp->rq_entry);
610 	}
611 
612 	down_write(&qp->state_lock);
613 
614 	qp_attrs.state = SIW_QP_STATE_ERROR;
615 	siw_qp_modify(qp, &qp_attrs, SIW_QP_ATTR_STATE);
616 
617 	if (qp->cep) {
618 		siw_cep_put(qp->cep);
619 		qp->cep = NULL;
620 	}
621 	up_write(&qp->state_lock);
622 
623 	kfree(qp->tx_ctx.mpa_crc_hd);
624 	kfree(qp->rx_stream.mpa_crc_hd);
625 
626 	qp->scq = qp->rcq = NULL;
627 
628 	siw_qp_put(qp);
629 	wait_for_completion(&qp->qp_free);
630 
631 	return 0;
632 }
633 
634 /*
635  * siw_copy_inline_sgl()
636  *
637  * Prepare sgl of inlined data for sending. For userland callers
638  * function checks if given buffer addresses and len's are within
639  * process context bounds.
640  * Data from all provided sge's are copied together into the wqe,
641  * referenced by a single sge.
642  */
siw_copy_inline_sgl(const struct ib_send_wr * core_wr,struct siw_sqe * sqe)643 static int siw_copy_inline_sgl(const struct ib_send_wr *core_wr,
644 			       struct siw_sqe *sqe)
645 {
646 	struct ib_sge *core_sge = core_wr->sg_list;
647 	void *kbuf = &sqe->sge[1];
648 	int num_sge = core_wr->num_sge, bytes = 0;
649 
650 	sqe->sge[0].laddr = (uintptr_t)kbuf;
651 	sqe->sge[0].lkey = 0;
652 
653 	while (num_sge--) {
654 		if (!core_sge->length) {
655 			core_sge++;
656 			continue;
657 		}
658 		bytes += core_sge->length;
659 		if (bytes > SIW_MAX_INLINE) {
660 			bytes = -EINVAL;
661 			break;
662 		}
663 		memcpy(kbuf, (void *)(uintptr_t)core_sge->addr,
664 		       core_sge->length);
665 
666 		kbuf += core_sge->length;
667 		core_sge++;
668 	}
669 	sqe->sge[0].length = max(bytes, 0);
670 	sqe->num_sge = bytes > 0 ? 1 : 0;
671 
672 	return bytes;
673 }
674 
675 /* Complete SQ WR's without processing */
siw_sq_flush_wr(struct siw_qp * qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)676 static int siw_sq_flush_wr(struct siw_qp *qp, const struct ib_send_wr *wr,
677 			   const struct ib_send_wr **bad_wr)
678 {
679 	int rv = 0;
680 
681 	while (wr) {
682 		struct siw_sqe sqe = {};
683 
684 		switch (wr->opcode) {
685 		case IB_WR_RDMA_WRITE:
686 			sqe.opcode = SIW_OP_WRITE;
687 			break;
688 		case IB_WR_RDMA_READ:
689 			sqe.opcode = SIW_OP_READ;
690 			break;
691 		case IB_WR_RDMA_READ_WITH_INV:
692 			sqe.opcode = SIW_OP_READ_LOCAL_INV;
693 			break;
694 		case IB_WR_SEND:
695 			sqe.opcode = SIW_OP_SEND;
696 			break;
697 		case IB_WR_SEND_WITH_IMM:
698 			sqe.opcode = SIW_OP_SEND_WITH_IMM;
699 			break;
700 		case IB_WR_SEND_WITH_INV:
701 			sqe.opcode = SIW_OP_SEND_REMOTE_INV;
702 			break;
703 		case IB_WR_LOCAL_INV:
704 			sqe.opcode = SIW_OP_INVAL_STAG;
705 			break;
706 		case IB_WR_REG_MR:
707 			sqe.opcode = SIW_OP_REG_MR;
708 			break;
709 		default:
710 			rv = -EINVAL;
711 			break;
712 		}
713 		if (!rv) {
714 			sqe.id = wr->wr_id;
715 			rv = siw_sqe_complete(qp, &sqe, 0,
716 					      SIW_WC_WR_FLUSH_ERR);
717 		}
718 		if (rv) {
719 			if (bad_wr)
720 				*bad_wr = wr;
721 			break;
722 		}
723 		wr = wr->next;
724 	}
725 	return rv;
726 }
727 
728 /* Complete RQ WR's without processing */
siw_rq_flush_wr(struct siw_qp * qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)729 static int siw_rq_flush_wr(struct siw_qp *qp, const struct ib_recv_wr *wr,
730 			   const struct ib_recv_wr **bad_wr)
731 {
732 	struct siw_rqe rqe = {};
733 	int rv = 0;
734 
735 	while (wr) {
736 		rqe.id = wr->wr_id;
737 		rv = siw_rqe_complete(qp, &rqe, 0, 0, SIW_WC_WR_FLUSH_ERR);
738 		if (rv) {
739 			if (bad_wr)
740 				*bad_wr = wr;
741 			break;
742 		}
743 		wr = wr->next;
744 	}
745 	return rv;
746 }
747 
748 /*
749  * siw_post_send()
750  *
751  * Post a list of S-WR's to a SQ.
752  *
753  * @base_qp:	Base QP contained in siw QP
754  * @wr:		Null terminated list of user WR's
755  * @bad_wr:	Points to failing WR in case of synchronous failure.
756  */
siw_post_send(struct ib_qp * base_qp,const struct ib_send_wr * wr,const struct ib_send_wr ** bad_wr)757 int siw_post_send(struct ib_qp *base_qp, const struct ib_send_wr *wr,
758 		  const struct ib_send_wr **bad_wr)
759 {
760 	struct siw_qp *qp = to_siw_qp(base_qp);
761 	struct siw_wqe *wqe = tx_wqe(qp);
762 
763 	unsigned long flags;
764 	int rv = 0;
765 
766 	if (wr && !rdma_is_kernel_res(&qp->base_qp.res)) {
767 		siw_dbg_qp(qp, "wr must be empty for user mapped sq\n");
768 		*bad_wr = wr;
769 		return -EINVAL;
770 	}
771 
772 	/*
773 	 * Try to acquire QP state lock. Must be non-blocking
774 	 * to accommodate kernel clients needs.
775 	 */
776 	if (!down_read_trylock(&qp->state_lock)) {
777 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
778 			/*
779 			 * ERROR state is final, so we can be sure
780 			 * this state will not change as long as the QP
781 			 * exists.
782 			 *
783 			 * This handles an ib_drain_sq() call with
784 			 * a concurrent request to set the QP state
785 			 * to ERROR.
786 			 */
787 			rv = siw_sq_flush_wr(qp, wr, bad_wr);
788 		} else {
789 			siw_dbg_qp(qp, "QP locked, state %d\n",
790 				   qp->attrs.state);
791 			*bad_wr = wr;
792 			rv = -ENOTCONN;
793 		}
794 		return rv;
795 	}
796 	if (unlikely(qp->attrs.state != SIW_QP_STATE_RTS)) {
797 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
798 			/*
799 			 * Immediately flush this WR to CQ, if QP
800 			 * is in ERROR state. SQ is guaranteed to
801 			 * be empty, so WR complets in-order.
802 			 *
803 			 * Typically triggered by ib_drain_sq().
804 			 */
805 			rv = siw_sq_flush_wr(qp, wr, bad_wr);
806 		} else {
807 			siw_dbg_qp(qp, "QP out of state %d\n",
808 				   qp->attrs.state);
809 			*bad_wr = wr;
810 			rv = -ENOTCONN;
811 		}
812 		up_read(&qp->state_lock);
813 		return rv;
814 	}
815 	spin_lock_irqsave(&qp->sq_lock, flags);
816 
817 	while (wr) {
818 		u32 idx = qp->sq_put % qp->attrs.sq_size;
819 		struct siw_sqe *sqe = &qp->sendq[idx];
820 
821 		if (sqe->flags) {
822 			siw_dbg_qp(qp, "sq full\n");
823 			rv = -ENOMEM;
824 			break;
825 		}
826 		if (wr->num_sge > qp->attrs.sq_max_sges) {
827 			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
828 			rv = -EINVAL;
829 			break;
830 		}
831 		sqe->id = wr->wr_id;
832 
833 		if ((wr->send_flags & IB_SEND_SIGNALED) ||
834 		    (qp->attrs.flags & SIW_SIGNAL_ALL_WR))
835 			sqe->flags |= SIW_WQE_SIGNALLED;
836 
837 		if (wr->send_flags & IB_SEND_FENCE)
838 			sqe->flags |= SIW_WQE_READ_FENCE;
839 
840 		switch (wr->opcode) {
841 		case IB_WR_SEND:
842 		case IB_WR_SEND_WITH_INV:
843 			if (wr->send_flags & IB_SEND_SOLICITED)
844 				sqe->flags |= SIW_WQE_SOLICITED;
845 
846 			if (!(wr->send_flags & IB_SEND_INLINE)) {
847 				siw_copy_sgl(wr->sg_list, sqe->sge,
848 					     wr->num_sge);
849 				sqe->num_sge = wr->num_sge;
850 			} else {
851 				rv = siw_copy_inline_sgl(wr, sqe);
852 				if (rv <= 0) {
853 					rv = -EINVAL;
854 					break;
855 				}
856 				sqe->flags |= SIW_WQE_INLINE;
857 				sqe->num_sge = 1;
858 			}
859 			if (wr->opcode == IB_WR_SEND)
860 				sqe->opcode = SIW_OP_SEND;
861 			else {
862 				sqe->opcode = SIW_OP_SEND_REMOTE_INV;
863 				sqe->rkey = wr->ex.invalidate_rkey;
864 			}
865 			break;
866 
867 		case IB_WR_RDMA_READ_WITH_INV:
868 		case IB_WR_RDMA_READ:
869 			/*
870 			 * iWarp restricts RREAD sink to SGL containing
871 			 * 1 SGE only. we could relax to SGL with multiple
872 			 * elements referring the SAME ltag or even sending
873 			 * a private per-rreq tag referring to a checked
874 			 * local sgl with MULTIPLE ltag's.
875 			 */
876 			if (unlikely(wr->num_sge != 1)) {
877 				rv = -EINVAL;
878 				break;
879 			}
880 			siw_copy_sgl(wr->sg_list, &sqe->sge[0], 1);
881 			/*
882 			 * NOTE: zero length RREAD is allowed!
883 			 */
884 			sqe->raddr = rdma_wr(wr)->remote_addr;
885 			sqe->rkey = rdma_wr(wr)->rkey;
886 			sqe->num_sge = 1;
887 
888 			if (wr->opcode == IB_WR_RDMA_READ)
889 				sqe->opcode = SIW_OP_READ;
890 			else
891 				sqe->opcode = SIW_OP_READ_LOCAL_INV;
892 			break;
893 
894 		case IB_WR_RDMA_WRITE:
895 			if (!(wr->send_flags & IB_SEND_INLINE)) {
896 				siw_copy_sgl(wr->sg_list, &sqe->sge[0],
897 					     wr->num_sge);
898 				sqe->num_sge = wr->num_sge;
899 			} else {
900 				rv = siw_copy_inline_sgl(wr, sqe);
901 				if (unlikely(rv < 0)) {
902 					rv = -EINVAL;
903 					break;
904 				}
905 				sqe->flags |= SIW_WQE_INLINE;
906 				sqe->num_sge = 1;
907 			}
908 			sqe->raddr = rdma_wr(wr)->remote_addr;
909 			sqe->rkey = rdma_wr(wr)->rkey;
910 			sqe->opcode = SIW_OP_WRITE;
911 			break;
912 
913 		case IB_WR_REG_MR:
914 			sqe->base_mr = (uintptr_t)reg_wr(wr)->mr;
915 			sqe->rkey = reg_wr(wr)->key;
916 			sqe->access = reg_wr(wr)->access & IWARP_ACCESS_MASK;
917 			sqe->opcode = SIW_OP_REG_MR;
918 			break;
919 
920 		case IB_WR_LOCAL_INV:
921 			sqe->rkey = wr->ex.invalidate_rkey;
922 			sqe->opcode = SIW_OP_INVAL_STAG;
923 			break;
924 
925 		default:
926 			siw_dbg_qp(qp, "ib wr type %d unsupported\n",
927 				   wr->opcode);
928 			rv = -EINVAL;
929 			break;
930 		}
931 		siw_dbg_qp(qp, "opcode %d, flags 0x%x, wr_id 0x%pK\n",
932 			   sqe->opcode, sqe->flags,
933 			   (void *)(uintptr_t)sqe->id);
934 
935 		if (unlikely(rv < 0))
936 			break;
937 
938 		/* make SQE only valid after completely written */
939 		smp_wmb();
940 		sqe->flags |= SIW_WQE_VALID;
941 
942 		qp->sq_put++;
943 		wr = wr->next;
944 	}
945 
946 	/*
947 	 * Send directly if SQ processing is not in progress.
948 	 * Eventual immediate errors (rv < 0) do not affect the involved
949 	 * RI resources (Verbs, 8.3.1) and thus do not prevent from SQ
950 	 * processing, if new work is already pending. But rv must be passed
951 	 * to caller.
952 	 */
953 	if (wqe->wr_status != SIW_WR_IDLE) {
954 		spin_unlock_irqrestore(&qp->sq_lock, flags);
955 		goto skip_direct_sending;
956 	}
957 	rv = siw_activate_tx(qp);
958 	spin_unlock_irqrestore(&qp->sq_lock, flags);
959 
960 	if (rv <= 0)
961 		goto skip_direct_sending;
962 
963 	if (rdma_is_kernel_res(&qp->base_qp.res)) {
964 		rv = siw_sq_start(qp);
965 	} else {
966 		qp->tx_ctx.in_syscall = 1;
967 
968 		if (siw_qp_sq_process(qp) != 0 && !(qp->tx_ctx.tx_suspend))
969 			siw_qp_cm_drop(qp, 0);
970 
971 		qp->tx_ctx.in_syscall = 0;
972 	}
973 skip_direct_sending:
974 
975 	up_read(&qp->state_lock);
976 
977 	if (rv >= 0)
978 		return 0;
979 	/*
980 	 * Immediate error
981 	 */
982 	siw_dbg_qp(qp, "error %d\n", rv);
983 
984 	*bad_wr = wr;
985 	return rv;
986 }
987 
988 /*
989  * siw_post_receive()
990  *
991  * Post a list of R-WR's to a RQ.
992  *
993  * @base_qp:	Base QP contained in siw QP
994  * @wr:		Null terminated list of user WR's
995  * @bad_wr:	Points to failing WR in case of synchronous failure.
996  */
siw_post_receive(struct ib_qp * base_qp,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)997 int siw_post_receive(struct ib_qp *base_qp, const struct ib_recv_wr *wr,
998 		     const struct ib_recv_wr **bad_wr)
999 {
1000 	struct siw_qp *qp = to_siw_qp(base_qp);
1001 	unsigned long flags;
1002 	int rv = 0;
1003 
1004 	if (qp->srq || qp->attrs.rq_size == 0) {
1005 		*bad_wr = wr;
1006 		return -EINVAL;
1007 	}
1008 	if (!rdma_is_kernel_res(&qp->base_qp.res)) {
1009 		siw_dbg_qp(qp, "no kernel post_recv for user mapped rq\n");
1010 		*bad_wr = wr;
1011 		return -EINVAL;
1012 	}
1013 
1014 	/*
1015 	 * Try to acquire QP state lock. Must be non-blocking
1016 	 * to accommodate kernel clients needs.
1017 	 */
1018 	if (!down_read_trylock(&qp->state_lock)) {
1019 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1020 			/*
1021 			 * ERROR state is final, so we can be sure
1022 			 * this state will not change as long as the QP
1023 			 * exists.
1024 			 *
1025 			 * This handles an ib_drain_rq() call with
1026 			 * a concurrent request to set the QP state
1027 			 * to ERROR.
1028 			 */
1029 			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1030 		} else {
1031 			siw_dbg_qp(qp, "QP locked, state %d\n",
1032 				   qp->attrs.state);
1033 			*bad_wr = wr;
1034 			rv = -ENOTCONN;
1035 		}
1036 		return rv;
1037 	}
1038 	if (qp->attrs.state > SIW_QP_STATE_RTS) {
1039 		if (qp->attrs.state == SIW_QP_STATE_ERROR) {
1040 			/*
1041 			 * Immediately flush this WR to CQ, if QP
1042 			 * is in ERROR state. RQ is guaranteed to
1043 			 * be empty, so WR complets in-order.
1044 			 *
1045 			 * Typically triggered by ib_drain_rq().
1046 			 */
1047 			rv = siw_rq_flush_wr(qp, wr, bad_wr);
1048 		} else {
1049 			siw_dbg_qp(qp, "QP out of state %d\n",
1050 				   qp->attrs.state);
1051 			*bad_wr = wr;
1052 			rv = -ENOTCONN;
1053 		}
1054 		up_read(&qp->state_lock);
1055 		return rv;
1056 	}
1057 	/*
1058 	 * Serialize potentially multiple producers.
1059 	 * Not needed for single threaded consumer side.
1060 	 */
1061 	spin_lock_irqsave(&qp->rq_lock, flags);
1062 
1063 	while (wr) {
1064 		u32 idx = qp->rq_put % qp->attrs.rq_size;
1065 		struct siw_rqe *rqe = &qp->recvq[idx];
1066 
1067 		if (rqe->flags) {
1068 			siw_dbg_qp(qp, "RQ full\n");
1069 			rv = -ENOMEM;
1070 			break;
1071 		}
1072 		if (wr->num_sge > qp->attrs.rq_max_sges) {
1073 			siw_dbg_qp(qp, "too many sge's: %d\n", wr->num_sge);
1074 			rv = -EINVAL;
1075 			break;
1076 		}
1077 		rqe->id = wr->wr_id;
1078 		rqe->num_sge = wr->num_sge;
1079 		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1080 
1081 		/* make sure RQE is completely written before valid */
1082 		smp_wmb();
1083 
1084 		rqe->flags = SIW_WQE_VALID;
1085 
1086 		qp->rq_put++;
1087 		wr = wr->next;
1088 	}
1089 	spin_unlock_irqrestore(&qp->rq_lock, flags);
1090 
1091 	up_read(&qp->state_lock);
1092 
1093 	if (rv < 0) {
1094 		siw_dbg_qp(qp, "error %d\n", rv);
1095 		*bad_wr = wr;
1096 	}
1097 	return rv > 0 ? 0 : rv;
1098 }
1099 
siw_destroy_cq(struct ib_cq * base_cq,struct ib_udata * udata)1100 int siw_destroy_cq(struct ib_cq *base_cq, struct ib_udata *udata)
1101 {
1102 	struct siw_cq *cq = to_siw_cq(base_cq);
1103 	struct siw_device *sdev = to_siw_dev(base_cq->device);
1104 	struct siw_ucontext *ctx =
1105 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1106 					  base_ucontext);
1107 
1108 	siw_dbg_cq(cq, "free CQ resources\n");
1109 
1110 	siw_cq_flush(cq);
1111 
1112 	if (ctx)
1113 		rdma_user_mmap_entry_remove(cq->cq_entry);
1114 
1115 	atomic_dec(&sdev->num_cq);
1116 
1117 	vfree(cq->queue);
1118 	return 0;
1119 }
1120 
1121 /*
1122  * siw_create_cq()
1123  *
1124  * Populate CQ of requested size
1125  *
1126  * @base_cq: CQ as allocated by RDMA midlayer
1127  * @attr: Initial CQ attributes
1128  * @udata: relates to user context
1129  */
1130 
siw_create_cq(struct ib_cq * base_cq,const struct ib_cq_init_attr * attr,struct ib_udata * udata)1131 int siw_create_cq(struct ib_cq *base_cq, const struct ib_cq_init_attr *attr,
1132 		  struct ib_udata *udata)
1133 {
1134 	struct siw_device *sdev = to_siw_dev(base_cq->device);
1135 	struct siw_cq *cq = to_siw_cq(base_cq);
1136 	int rv, size = attr->cqe;
1137 
1138 	if (attr->flags)
1139 		return -EOPNOTSUPP;
1140 
1141 	if (atomic_inc_return(&sdev->num_cq) > SIW_MAX_CQ) {
1142 		siw_dbg(base_cq->device, "too many CQ's\n");
1143 		rv = -ENOMEM;
1144 		goto err_out;
1145 	}
1146 	if (size < 1 || size > sdev->attrs.max_cqe) {
1147 		siw_dbg(base_cq->device, "CQ size error: %d\n", size);
1148 		rv = -EINVAL;
1149 		goto err_out;
1150 	}
1151 	size = roundup_pow_of_two(size);
1152 	cq->base_cq.cqe = size;
1153 	cq->num_cqe = size;
1154 
1155 	if (udata)
1156 		cq->queue = vmalloc_user(size * sizeof(struct siw_cqe) +
1157 					 sizeof(struct siw_cq_ctrl));
1158 	else
1159 		cq->queue = vzalloc(size * sizeof(struct siw_cqe) +
1160 				    sizeof(struct siw_cq_ctrl));
1161 
1162 	if (cq->queue == NULL) {
1163 		rv = -ENOMEM;
1164 		goto err_out;
1165 	}
1166 	get_random_bytes(&cq->id, 4);
1167 	siw_dbg(base_cq->device, "new CQ [%u]\n", cq->id);
1168 
1169 	spin_lock_init(&cq->lock);
1170 
1171 	cq->notify = (struct siw_cq_ctrl *)&cq->queue[size];
1172 
1173 	if (udata) {
1174 		struct siw_uresp_create_cq uresp = {};
1175 		struct siw_ucontext *ctx =
1176 			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1177 						  base_ucontext);
1178 		size_t length = size * sizeof(struct siw_cqe) +
1179 			sizeof(struct siw_cq_ctrl);
1180 
1181 		cq->cq_entry =
1182 			siw_mmap_entry_insert(ctx, cq->queue,
1183 					      length, &uresp.cq_key);
1184 		if (!cq->cq_entry) {
1185 			rv = -ENOMEM;
1186 			goto err_out;
1187 		}
1188 
1189 		uresp.cq_id = cq->id;
1190 		uresp.num_cqe = size;
1191 
1192 		if (udata->outlen < sizeof(uresp)) {
1193 			rv = -EINVAL;
1194 			goto err_out;
1195 		}
1196 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1197 		if (rv)
1198 			goto err_out;
1199 	}
1200 	return 0;
1201 
1202 err_out:
1203 	siw_dbg(base_cq->device, "CQ creation failed: %d", rv);
1204 
1205 	if (cq->queue) {
1206 		struct siw_ucontext *ctx =
1207 			rdma_udata_to_drv_context(udata, struct siw_ucontext,
1208 						  base_ucontext);
1209 		if (ctx)
1210 			rdma_user_mmap_entry_remove(cq->cq_entry);
1211 		vfree(cq->queue);
1212 	}
1213 	atomic_dec(&sdev->num_cq);
1214 
1215 	return rv;
1216 }
1217 
1218 /*
1219  * siw_poll_cq()
1220  *
1221  * Reap CQ entries if available and copy work completion status into
1222  * array of WC's provided by caller. Returns number of reaped CQE's.
1223  *
1224  * @base_cq:	Base CQ contained in siw CQ.
1225  * @num_cqe:	Maximum number of CQE's to reap.
1226  * @wc:		Array of work completions to be filled by siw.
1227  */
siw_poll_cq(struct ib_cq * base_cq,int num_cqe,struct ib_wc * wc)1228 int siw_poll_cq(struct ib_cq *base_cq, int num_cqe, struct ib_wc *wc)
1229 {
1230 	struct siw_cq *cq = to_siw_cq(base_cq);
1231 	int i;
1232 
1233 	for (i = 0; i < num_cqe; i++) {
1234 		if (!siw_reap_cqe(cq, wc))
1235 			break;
1236 		wc++;
1237 	}
1238 	return i;
1239 }
1240 
1241 /*
1242  * siw_req_notify_cq()
1243  *
1244  * Request notification for new CQE's added to that CQ.
1245  * Defined flags:
1246  * o SIW_CQ_NOTIFY_SOLICITED lets siw trigger a notification
1247  *   event if a WQE with notification flag set enters the CQ
1248  * o SIW_CQ_NOTIFY_NEXT_COMP lets siw trigger a notification
1249  *   event if a WQE enters the CQ.
1250  * o IB_CQ_REPORT_MISSED_EVENTS: return value will provide the
1251  *   number of not reaped CQE's regardless of its notification
1252  *   type and current or new CQ notification settings.
1253  *
1254  * @base_cq:	Base CQ contained in siw CQ.
1255  * @flags:	Requested notification flags.
1256  */
siw_req_notify_cq(struct ib_cq * base_cq,enum ib_cq_notify_flags flags)1257 int siw_req_notify_cq(struct ib_cq *base_cq, enum ib_cq_notify_flags flags)
1258 {
1259 	struct siw_cq *cq = to_siw_cq(base_cq);
1260 
1261 	siw_dbg_cq(cq, "flags: 0x%02x\n", flags);
1262 
1263 	if ((flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED)
1264 		/*
1265 		 * Enable CQ event for next solicited completion.
1266 		 * and make it visible to all associated producers.
1267 		 */
1268 		smp_store_mb(cq->notify->flags, SIW_NOTIFY_SOLICITED);
1269 	else
1270 		/*
1271 		 * Enable CQ event for any signalled completion.
1272 		 * and make it visible to all associated producers.
1273 		 */
1274 		smp_store_mb(cq->notify->flags, SIW_NOTIFY_ALL);
1275 
1276 	if (flags & IB_CQ_REPORT_MISSED_EVENTS)
1277 		return cq->cq_put - cq->cq_get;
1278 
1279 	return 0;
1280 }
1281 
1282 /*
1283  * siw_dereg_mr()
1284  *
1285  * Release Memory Region.
1286  *
1287  * @base_mr: Base MR contained in siw MR.
1288  * @udata: points to user context, unused.
1289  */
siw_dereg_mr(struct ib_mr * base_mr,struct ib_udata * udata)1290 int siw_dereg_mr(struct ib_mr *base_mr, struct ib_udata *udata)
1291 {
1292 	struct siw_mr *mr = to_siw_mr(base_mr);
1293 	struct siw_device *sdev = to_siw_dev(base_mr->device);
1294 
1295 	siw_dbg_mem(mr->mem, "deregister MR\n");
1296 
1297 	atomic_dec(&sdev->num_mr);
1298 
1299 	siw_mr_drop_mem(mr);
1300 	kfree_rcu(mr, rcu);
1301 
1302 	return 0;
1303 }
1304 
1305 /*
1306  * siw_reg_user_mr()
1307  *
1308  * Register Memory Region.
1309  *
1310  * @pd:		Protection Domain
1311  * @start:	starting address of MR (virtual address)
1312  * @len:	len of MR
1313  * @rnic_va:	not used by siw
1314  * @rights:	MR access rights
1315  * @udata:	user buffer to communicate STag and Key.
1316  */
siw_reg_user_mr(struct ib_pd * pd,u64 start,u64 len,u64 rnic_va,int rights,struct ib_udata * udata)1317 struct ib_mr *siw_reg_user_mr(struct ib_pd *pd, u64 start, u64 len,
1318 			      u64 rnic_va, int rights, struct ib_udata *udata)
1319 {
1320 	struct siw_mr *mr = NULL;
1321 	struct siw_umem *umem = NULL;
1322 	struct siw_ureq_reg_mr ureq;
1323 	struct siw_device *sdev = to_siw_dev(pd->device);
1324 
1325 	unsigned long mem_limit = rlimit(RLIMIT_MEMLOCK);
1326 	int rv;
1327 
1328 	siw_dbg_pd(pd, "start: 0x%pK, va: 0x%pK, len: %llu\n",
1329 		   (void *)(uintptr_t)start, (void *)(uintptr_t)rnic_va,
1330 		   (unsigned long long)len);
1331 
1332 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1333 		siw_dbg_pd(pd, "too many mr's\n");
1334 		rv = -ENOMEM;
1335 		goto err_out;
1336 	}
1337 	if (!len) {
1338 		rv = -EINVAL;
1339 		goto err_out;
1340 	}
1341 	if (mem_limit != RLIM_INFINITY) {
1342 		unsigned long num_pages =
1343 			(PAGE_ALIGN(len + (start & ~PAGE_MASK))) >> PAGE_SHIFT;
1344 		mem_limit >>= PAGE_SHIFT;
1345 
1346 		if (num_pages > mem_limit - current->mm->locked_vm) {
1347 			siw_dbg_pd(pd, "pages req %lu, max %lu, lock %lu\n",
1348 				   num_pages, mem_limit,
1349 				   current->mm->locked_vm);
1350 			rv = -ENOMEM;
1351 			goto err_out;
1352 		}
1353 	}
1354 	umem = siw_umem_get(start, len, ib_access_writable(rights));
1355 	if (IS_ERR(umem)) {
1356 		rv = PTR_ERR(umem);
1357 		siw_dbg_pd(pd, "getting user memory failed: %d\n", rv);
1358 		umem = NULL;
1359 		goto err_out;
1360 	}
1361 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1362 	if (!mr) {
1363 		rv = -ENOMEM;
1364 		goto err_out;
1365 	}
1366 	rv = siw_mr_add_mem(mr, pd, umem, start, len, rights);
1367 	if (rv)
1368 		goto err_out;
1369 
1370 	if (udata) {
1371 		struct siw_uresp_reg_mr uresp = {};
1372 		struct siw_mem *mem = mr->mem;
1373 
1374 		if (udata->inlen < sizeof(ureq)) {
1375 			rv = -EINVAL;
1376 			goto err_out;
1377 		}
1378 		rv = ib_copy_from_udata(&ureq, udata, sizeof(ureq));
1379 		if (rv)
1380 			goto err_out;
1381 
1382 		mr->base_mr.lkey |= ureq.stag_key;
1383 		mr->base_mr.rkey |= ureq.stag_key;
1384 		mem->stag |= ureq.stag_key;
1385 		uresp.stag = mem->stag;
1386 
1387 		if (udata->outlen < sizeof(uresp)) {
1388 			rv = -EINVAL;
1389 			goto err_out;
1390 		}
1391 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1392 		if (rv)
1393 			goto err_out;
1394 	}
1395 	mr->mem->stag_valid = 1;
1396 
1397 	return &mr->base_mr;
1398 
1399 err_out:
1400 	atomic_dec(&sdev->num_mr);
1401 	if (mr) {
1402 		if (mr->mem)
1403 			siw_mr_drop_mem(mr);
1404 		kfree_rcu(mr, rcu);
1405 	} else {
1406 		if (umem)
1407 			siw_umem_release(umem, false);
1408 	}
1409 	return ERR_PTR(rv);
1410 }
1411 
siw_alloc_mr(struct ib_pd * pd,enum ib_mr_type mr_type,u32 max_sge)1412 struct ib_mr *siw_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type,
1413 			   u32 max_sge)
1414 {
1415 	struct siw_device *sdev = to_siw_dev(pd->device);
1416 	struct siw_mr *mr = NULL;
1417 	struct siw_pbl *pbl = NULL;
1418 	int rv;
1419 
1420 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1421 		siw_dbg_pd(pd, "too many mr's\n");
1422 		rv = -ENOMEM;
1423 		goto err_out;
1424 	}
1425 	if (mr_type != IB_MR_TYPE_MEM_REG) {
1426 		siw_dbg_pd(pd, "mr type %d unsupported\n", mr_type);
1427 		rv = -EOPNOTSUPP;
1428 		goto err_out;
1429 	}
1430 	if (max_sge > SIW_MAX_SGE_PBL) {
1431 		siw_dbg_pd(pd, "too many sge's: %d\n", max_sge);
1432 		rv = -ENOMEM;
1433 		goto err_out;
1434 	}
1435 	pbl = siw_pbl_alloc(max_sge);
1436 	if (IS_ERR(pbl)) {
1437 		rv = PTR_ERR(pbl);
1438 		siw_dbg_pd(pd, "pbl allocation failed: %d\n", rv);
1439 		pbl = NULL;
1440 		goto err_out;
1441 	}
1442 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1443 	if (!mr) {
1444 		rv = -ENOMEM;
1445 		goto err_out;
1446 	}
1447 	rv = siw_mr_add_mem(mr, pd, pbl, 0, max_sge * PAGE_SIZE, 0);
1448 	if (rv)
1449 		goto err_out;
1450 
1451 	mr->mem->is_pbl = 1;
1452 
1453 	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1454 
1455 	return &mr->base_mr;
1456 
1457 err_out:
1458 	atomic_dec(&sdev->num_mr);
1459 
1460 	if (!mr) {
1461 		kfree(pbl);
1462 	} else {
1463 		if (mr->mem)
1464 			siw_mr_drop_mem(mr);
1465 		kfree_rcu(mr, rcu);
1466 	}
1467 	siw_dbg_pd(pd, "failed: %d\n", rv);
1468 
1469 	return ERR_PTR(rv);
1470 }
1471 
1472 /* Just used to count number of pages being mapped */
siw_set_pbl_page(struct ib_mr * base_mr,u64 buf_addr)1473 static int siw_set_pbl_page(struct ib_mr *base_mr, u64 buf_addr)
1474 {
1475 	return 0;
1476 }
1477 
siw_map_mr_sg(struct ib_mr * base_mr,struct scatterlist * sl,int num_sle,unsigned int * sg_off)1478 int siw_map_mr_sg(struct ib_mr *base_mr, struct scatterlist *sl, int num_sle,
1479 		  unsigned int *sg_off)
1480 {
1481 	struct scatterlist *slp;
1482 	struct siw_mr *mr = to_siw_mr(base_mr);
1483 	struct siw_mem *mem = mr->mem;
1484 	struct siw_pbl *pbl = mem->pbl;
1485 	struct siw_pble *pble;
1486 	unsigned long pbl_size;
1487 	int i, rv;
1488 
1489 	if (!pbl) {
1490 		siw_dbg_mem(mem, "no PBL allocated\n");
1491 		return -EINVAL;
1492 	}
1493 	pble = pbl->pbe;
1494 
1495 	if (pbl->max_buf < num_sle) {
1496 		siw_dbg_mem(mem, "too many SGE's: %d > %d\n",
1497 			    mem->pbl->max_buf, num_sle);
1498 		return -ENOMEM;
1499 	}
1500 	for_each_sg(sl, slp, num_sle, i) {
1501 		if (sg_dma_len(slp) == 0) {
1502 			siw_dbg_mem(mem, "empty SGE\n");
1503 			return -EINVAL;
1504 		}
1505 		if (i == 0) {
1506 			pble->addr = sg_dma_address(slp);
1507 			pble->size = sg_dma_len(slp);
1508 			pble->pbl_off = 0;
1509 			pbl_size = pble->size;
1510 			pbl->num_buf = 1;
1511 		} else {
1512 			/* Merge PBL entries if adjacent */
1513 			if (pble->addr + pble->size == sg_dma_address(slp)) {
1514 				pble->size += sg_dma_len(slp);
1515 			} else {
1516 				pble++;
1517 				pbl->num_buf++;
1518 				pble->addr = sg_dma_address(slp);
1519 				pble->size = sg_dma_len(slp);
1520 				pble->pbl_off = pbl_size;
1521 			}
1522 			pbl_size += sg_dma_len(slp);
1523 		}
1524 		siw_dbg_mem(mem,
1525 			"sge[%d], size %u, addr 0x%p, total %lu\n",
1526 			i, pble->size, (void *)(uintptr_t)pble->addr,
1527 			pbl_size);
1528 	}
1529 	rv = ib_sg_to_pages(base_mr, sl, num_sle, sg_off, siw_set_pbl_page);
1530 	if (rv > 0) {
1531 		mem->len = base_mr->length;
1532 		mem->va = base_mr->iova;
1533 		siw_dbg_mem(mem,
1534 			"%llu bytes, start 0x%pK, %u SLE to %u entries\n",
1535 			mem->len, (void *)(uintptr_t)mem->va, num_sle,
1536 			pbl->num_buf);
1537 	}
1538 	return rv;
1539 }
1540 
1541 /*
1542  * siw_get_dma_mr()
1543  *
1544  * Create a (empty) DMA memory region, where no umem is attached.
1545  */
siw_get_dma_mr(struct ib_pd * pd,int rights)1546 struct ib_mr *siw_get_dma_mr(struct ib_pd *pd, int rights)
1547 {
1548 	struct siw_device *sdev = to_siw_dev(pd->device);
1549 	struct siw_mr *mr = NULL;
1550 	int rv;
1551 
1552 	if (atomic_inc_return(&sdev->num_mr) > SIW_MAX_MR) {
1553 		siw_dbg_pd(pd, "too many mr's\n");
1554 		rv = -ENOMEM;
1555 		goto err_out;
1556 	}
1557 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
1558 	if (!mr) {
1559 		rv = -ENOMEM;
1560 		goto err_out;
1561 	}
1562 	rv = siw_mr_add_mem(mr, pd, NULL, 0, ULONG_MAX, rights);
1563 	if (rv)
1564 		goto err_out;
1565 
1566 	mr->mem->stag_valid = 1;
1567 
1568 	siw_dbg_pd(pd, "[MEM %u]: success\n", mr->mem->stag);
1569 
1570 	return &mr->base_mr;
1571 
1572 err_out:
1573 	if (rv)
1574 		kfree(mr);
1575 
1576 	atomic_dec(&sdev->num_mr);
1577 
1578 	return ERR_PTR(rv);
1579 }
1580 
1581 /*
1582  * siw_create_srq()
1583  *
1584  * Create Shared Receive Queue of attributes @init_attrs
1585  * within protection domain given by @pd.
1586  *
1587  * @base_srq:	Base SRQ contained in siw SRQ.
1588  * @init_attrs:	SRQ init attributes.
1589  * @udata:	points to user context
1590  */
siw_create_srq(struct ib_srq * base_srq,struct ib_srq_init_attr * init_attrs,struct ib_udata * udata)1591 int siw_create_srq(struct ib_srq *base_srq,
1592 		   struct ib_srq_init_attr *init_attrs, struct ib_udata *udata)
1593 {
1594 	struct siw_srq *srq = to_siw_srq(base_srq);
1595 	struct ib_srq_attr *attrs = &init_attrs->attr;
1596 	struct siw_device *sdev = to_siw_dev(base_srq->device);
1597 	struct siw_ucontext *ctx =
1598 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1599 					  base_ucontext);
1600 	int rv;
1601 
1602 	if (init_attrs->srq_type != IB_SRQT_BASIC)
1603 		return -EOPNOTSUPP;
1604 
1605 	if (atomic_inc_return(&sdev->num_srq) > SIW_MAX_SRQ) {
1606 		siw_dbg_pd(base_srq->pd, "too many SRQ's\n");
1607 		rv = -ENOMEM;
1608 		goto err_out;
1609 	}
1610 	if (attrs->max_wr == 0 || attrs->max_wr > SIW_MAX_SRQ_WR ||
1611 	    attrs->max_sge > SIW_MAX_SGE || attrs->srq_limit > attrs->max_wr) {
1612 		rv = -EINVAL;
1613 		goto err_out;
1614 	}
1615 	srq->max_sge = attrs->max_sge;
1616 	srq->num_rqe = roundup_pow_of_two(attrs->max_wr);
1617 	srq->limit = attrs->srq_limit;
1618 	if (srq->limit)
1619 		srq->armed = true;
1620 
1621 	srq->is_kernel_res = !udata;
1622 
1623 	if (udata)
1624 		srq->recvq =
1625 			vmalloc_user(srq->num_rqe * sizeof(struct siw_rqe));
1626 	else
1627 		srq->recvq = vzalloc(srq->num_rqe * sizeof(struct siw_rqe));
1628 
1629 	if (srq->recvq == NULL) {
1630 		rv = -ENOMEM;
1631 		goto err_out;
1632 	}
1633 	if (udata) {
1634 		struct siw_uresp_create_srq uresp = {};
1635 		size_t length = srq->num_rqe * sizeof(struct siw_rqe);
1636 
1637 		srq->srq_entry =
1638 			siw_mmap_entry_insert(ctx, srq->recvq,
1639 					      length, &uresp.srq_key);
1640 		if (!srq->srq_entry) {
1641 			rv = -ENOMEM;
1642 			goto err_out;
1643 		}
1644 
1645 		uresp.num_rqe = srq->num_rqe;
1646 
1647 		if (udata->outlen < sizeof(uresp)) {
1648 			rv = -EINVAL;
1649 			goto err_out;
1650 		}
1651 		rv = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1652 		if (rv)
1653 			goto err_out;
1654 	}
1655 	spin_lock_init(&srq->lock);
1656 
1657 	siw_dbg_pd(base_srq->pd, "[SRQ]: success\n");
1658 
1659 	return 0;
1660 
1661 err_out:
1662 	if (srq->recvq) {
1663 		if (ctx)
1664 			rdma_user_mmap_entry_remove(srq->srq_entry);
1665 		vfree(srq->recvq);
1666 	}
1667 	atomic_dec(&sdev->num_srq);
1668 
1669 	return rv;
1670 }
1671 
1672 /*
1673  * siw_modify_srq()
1674  *
1675  * Modify SRQ. The caller may resize SRQ and/or set/reset notification
1676  * limit and (re)arm IB_EVENT_SRQ_LIMIT_REACHED notification.
1677  *
1678  * NOTE: it is unclear if RDMA core allows for changing the MAX_SGE
1679  * parameter. siw_modify_srq() does not check the attrs->max_sge param.
1680  */
siw_modify_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs,enum ib_srq_attr_mask attr_mask,struct ib_udata * udata)1681 int siw_modify_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs,
1682 		   enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1683 {
1684 	struct siw_srq *srq = to_siw_srq(base_srq);
1685 	unsigned long flags;
1686 	int rv = 0;
1687 
1688 	spin_lock_irqsave(&srq->lock, flags);
1689 
1690 	if (attr_mask & IB_SRQ_MAX_WR) {
1691 		/* resize request not yet supported */
1692 		rv = -EOPNOTSUPP;
1693 		goto out;
1694 	}
1695 	if (attr_mask & IB_SRQ_LIMIT) {
1696 		if (attrs->srq_limit) {
1697 			if (unlikely(attrs->srq_limit > srq->num_rqe)) {
1698 				rv = -EINVAL;
1699 				goto out;
1700 			}
1701 			srq->armed = true;
1702 		} else {
1703 			srq->armed = false;
1704 		}
1705 		srq->limit = attrs->srq_limit;
1706 	}
1707 out:
1708 	spin_unlock_irqrestore(&srq->lock, flags);
1709 
1710 	return rv;
1711 }
1712 
1713 /*
1714  * siw_query_srq()
1715  *
1716  * Query SRQ attributes.
1717  */
siw_query_srq(struct ib_srq * base_srq,struct ib_srq_attr * attrs)1718 int siw_query_srq(struct ib_srq *base_srq, struct ib_srq_attr *attrs)
1719 {
1720 	struct siw_srq *srq = to_siw_srq(base_srq);
1721 	unsigned long flags;
1722 
1723 	spin_lock_irqsave(&srq->lock, flags);
1724 
1725 	attrs->max_wr = srq->num_rqe;
1726 	attrs->max_sge = srq->max_sge;
1727 	attrs->srq_limit = srq->limit;
1728 
1729 	spin_unlock_irqrestore(&srq->lock, flags);
1730 
1731 	return 0;
1732 }
1733 
1734 /*
1735  * siw_destroy_srq()
1736  *
1737  * Destroy SRQ.
1738  * It is assumed that the SRQ is not referenced by any
1739  * QP anymore - the code trusts the RDMA core environment to keep track
1740  * of QP references.
1741  */
siw_destroy_srq(struct ib_srq * base_srq,struct ib_udata * udata)1742 int siw_destroy_srq(struct ib_srq *base_srq, struct ib_udata *udata)
1743 {
1744 	struct siw_srq *srq = to_siw_srq(base_srq);
1745 	struct siw_device *sdev = to_siw_dev(base_srq->device);
1746 	struct siw_ucontext *ctx =
1747 		rdma_udata_to_drv_context(udata, struct siw_ucontext,
1748 					  base_ucontext);
1749 
1750 	if (ctx)
1751 		rdma_user_mmap_entry_remove(srq->srq_entry);
1752 	vfree(srq->recvq);
1753 	atomic_dec(&sdev->num_srq);
1754 	return 0;
1755 }
1756 
1757 /*
1758  * siw_post_srq_recv()
1759  *
1760  * Post a list of receive queue elements to SRQ.
1761  * NOTE: The function does not check or lock a certain SRQ state
1762  *       during the post operation. The code simply trusts the
1763  *       RDMA core environment.
1764  *
1765  * @base_srq:	Base SRQ contained in siw SRQ
1766  * @wr:		List of R-WR's
1767  * @bad_wr:	Updated to failing WR if posting fails.
1768  */
siw_post_srq_recv(struct ib_srq * base_srq,const struct ib_recv_wr * wr,const struct ib_recv_wr ** bad_wr)1769 int siw_post_srq_recv(struct ib_srq *base_srq, const struct ib_recv_wr *wr,
1770 		      const struct ib_recv_wr **bad_wr)
1771 {
1772 	struct siw_srq *srq = to_siw_srq(base_srq);
1773 	unsigned long flags;
1774 	int rv = 0;
1775 
1776 	if (unlikely(!srq->is_kernel_res)) {
1777 		siw_dbg_pd(base_srq->pd,
1778 			   "[SRQ]: no kernel post_recv for mapped srq\n");
1779 		rv = -EINVAL;
1780 		goto out;
1781 	}
1782 	/*
1783 	 * Serialize potentially multiple producers.
1784 	 * Also needed to serialize potentially multiple
1785 	 * consumers.
1786 	 */
1787 	spin_lock_irqsave(&srq->lock, flags);
1788 
1789 	while (wr) {
1790 		u32 idx = srq->rq_put % srq->num_rqe;
1791 		struct siw_rqe *rqe = &srq->recvq[idx];
1792 
1793 		if (rqe->flags) {
1794 			siw_dbg_pd(base_srq->pd, "SRQ full\n");
1795 			rv = -ENOMEM;
1796 			break;
1797 		}
1798 		if (unlikely(wr->num_sge > srq->max_sge)) {
1799 			siw_dbg_pd(base_srq->pd,
1800 				   "[SRQ]: too many sge's: %d\n", wr->num_sge);
1801 			rv = -EINVAL;
1802 			break;
1803 		}
1804 		rqe->id = wr->wr_id;
1805 		rqe->num_sge = wr->num_sge;
1806 		siw_copy_sgl(wr->sg_list, rqe->sge, wr->num_sge);
1807 
1808 		/* Make sure S-RQE is completely written before valid */
1809 		smp_wmb();
1810 
1811 		rqe->flags = SIW_WQE_VALID;
1812 
1813 		srq->rq_put++;
1814 		wr = wr->next;
1815 	}
1816 	spin_unlock_irqrestore(&srq->lock, flags);
1817 out:
1818 	if (unlikely(rv < 0)) {
1819 		siw_dbg_pd(base_srq->pd, "[SRQ]: error %d\n", rv);
1820 		*bad_wr = wr;
1821 	}
1822 	return rv;
1823 }
1824 
siw_qp_event(struct siw_qp * qp,enum ib_event_type etype)1825 void siw_qp_event(struct siw_qp *qp, enum ib_event_type etype)
1826 {
1827 	struct ib_event event;
1828 	struct ib_qp *base_qp = &qp->base_qp;
1829 
1830 	/*
1831 	 * Do not report asynchronous errors on QP which gets
1832 	 * destroyed via verbs interface (siw_destroy_qp())
1833 	 */
1834 	if (qp->attrs.flags & SIW_QP_IN_DESTROY)
1835 		return;
1836 
1837 	event.event = etype;
1838 	event.device = base_qp->device;
1839 	event.element.qp = base_qp;
1840 
1841 	if (base_qp->event_handler) {
1842 		siw_dbg_qp(qp, "reporting event %d\n", etype);
1843 		base_qp->event_handler(&event, base_qp->qp_context);
1844 	}
1845 }
1846 
siw_cq_event(struct siw_cq * cq,enum ib_event_type etype)1847 void siw_cq_event(struct siw_cq *cq, enum ib_event_type etype)
1848 {
1849 	struct ib_event event;
1850 	struct ib_cq *base_cq = &cq->base_cq;
1851 
1852 	event.event = etype;
1853 	event.device = base_cq->device;
1854 	event.element.cq = base_cq;
1855 
1856 	if (base_cq->event_handler) {
1857 		siw_dbg_cq(cq, "reporting CQ event %d\n", etype);
1858 		base_cq->event_handler(&event, base_cq->cq_context);
1859 	}
1860 }
1861 
siw_srq_event(struct siw_srq * srq,enum ib_event_type etype)1862 void siw_srq_event(struct siw_srq *srq, enum ib_event_type etype)
1863 {
1864 	struct ib_event event;
1865 	struct ib_srq *base_srq = &srq->base_srq;
1866 
1867 	event.event = etype;
1868 	event.device = base_srq->device;
1869 	event.element.srq = base_srq;
1870 
1871 	if (base_srq->event_handler) {
1872 		siw_dbg_pd(srq->base_srq.pd,
1873 			   "reporting SRQ event %d\n", etype);
1874 		base_srq->event_handler(&event, base_srq->srq_context);
1875 	}
1876 }
1877 
siw_port_event(struct siw_device * sdev,u32 port,enum ib_event_type etype)1878 void siw_port_event(struct siw_device *sdev, u32 port, enum ib_event_type etype)
1879 {
1880 	struct ib_event event;
1881 
1882 	event.event = etype;
1883 	event.device = &sdev->base_dev;
1884 	event.element.port_num = port;
1885 
1886 	siw_dbg(&sdev->base_dev, "reporting port event %d\n", etype);
1887 
1888 	ib_dispatch_event(&event);
1889 }
1890