1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Shared Memory Communications over RDMA (SMC-R) and RoCE
4  *
5  * Work Requests exploiting Infiniband API
6  *
7  * Work requests (WR) of type ib_post_send or ib_post_recv respectively
8  * are submitted to either RC SQ or RC RQ respectively
9  * (reliably connected send/receive queue)
10  * and become work queue entries (WQEs).
11  * While an SQ WR/WQE is pending, we track it until transmission completion.
12  * Through a send or receive completion queue (CQ) respectively,
13  * we get completion queue entries (CQEs) [aka work completions (WCs)].
14  * Since the CQ callback is called from IRQ context, we split work by using
15  * bottom halves implemented by tasklets.
16  *
17  * SMC uses this to exchange LLC (link layer control)
18  * and CDC (connection data control) messages.
19  *
20  * Copyright IBM Corp. 2016
21  *
22  * Author(s):  Steffen Maier <maier@linux.vnet.ibm.com>
23  */
24 
25 #include <linux/atomic.h>
26 #include <linux/hashtable.h>
27 #include <linux/wait.h>
28 #include <rdma/ib_verbs.h>
29 #include <asm/div64.h>
30 
31 #include "smc.h"
32 #include "smc_wr.h"
33 
34 #define SMC_WR_MAX_POLL_CQE 10	/* max. # of compl. queue elements in 1 poll */
35 
36 #define SMC_WR_RX_HASH_BITS 4
37 static DEFINE_HASHTABLE(smc_wr_rx_hash, SMC_WR_RX_HASH_BITS);
38 static DEFINE_SPINLOCK(smc_wr_rx_hash_lock);
39 
40 struct smc_wr_tx_pend {	/* control data for a pending send request */
41 	u64			wr_id;		/* work request id sent */
42 	smc_wr_tx_handler	handler;
43 	enum ib_wc_status	wc_status;	/* CQE status */
44 	struct smc_link		*link;
45 	u32			idx;
46 	struct smc_wr_tx_pend_priv priv;
47 	u8			compl_requested;
48 };
49 
50 /******************************** send queue *********************************/
51 
52 /*------------------------------- completion --------------------------------*/
53 
54 /* returns true if at least one tx work request is pending on the given link */
smc_wr_is_tx_pend(struct smc_link * link)55 static inline bool smc_wr_is_tx_pend(struct smc_link *link)
56 {
57 	return !bitmap_empty(link->wr_tx_mask, link->wr_tx_cnt);
58 }
59 
60 /* wait till all pending tx work requests on the given link are completed */
smc_wr_tx_wait_no_pending_sends(struct smc_link * link)61 void smc_wr_tx_wait_no_pending_sends(struct smc_link *link)
62 {
63 	wait_event(link->wr_tx_wait, !smc_wr_is_tx_pend(link));
64 }
65 
smc_wr_tx_find_pending_index(struct smc_link * link,u64 wr_id)66 static inline int smc_wr_tx_find_pending_index(struct smc_link *link, u64 wr_id)
67 {
68 	u32 i;
69 
70 	for (i = 0; i < link->wr_tx_cnt; i++) {
71 		if (link->wr_tx_pends[i].wr_id == wr_id)
72 			return i;
73 	}
74 	return link->wr_tx_cnt;
75 }
76 
smc_wr_tx_process_cqe(struct ib_wc * wc)77 static inline void smc_wr_tx_process_cqe(struct ib_wc *wc)
78 {
79 	struct smc_wr_tx_pend pnd_snd;
80 	struct smc_link *link;
81 	u32 pnd_snd_idx;
82 
83 	link = wc->qp->qp_context;
84 
85 	if (wc->opcode == IB_WC_REG_MR) {
86 		if (wc->status)
87 			link->wr_reg_state = FAILED;
88 		else
89 			link->wr_reg_state = CONFIRMED;
90 		smc_wr_wakeup_reg_wait(link);
91 		return;
92 	}
93 
94 	pnd_snd_idx = smc_wr_tx_find_pending_index(link, wc->wr_id);
95 	if (pnd_snd_idx == link->wr_tx_cnt) {
96 		if (link->lgr->smc_version != SMC_V2 ||
97 		    link->wr_tx_v2_pend->wr_id != wc->wr_id)
98 			return;
99 		link->wr_tx_v2_pend->wc_status = wc->status;
100 		memcpy(&pnd_snd, link->wr_tx_v2_pend, sizeof(pnd_snd));
101 		/* clear the full struct smc_wr_tx_pend including .priv */
102 		memset(link->wr_tx_v2_pend, 0,
103 		       sizeof(*link->wr_tx_v2_pend));
104 		memset(link->lgr->wr_tx_buf_v2, 0,
105 		       sizeof(*link->lgr->wr_tx_buf_v2));
106 	} else {
107 		link->wr_tx_pends[pnd_snd_idx].wc_status = wc->status;
108 		if (link->wr_tx_pends[pnd_snd_idx].compl_requested)
109 			complete(&link->wr_tx_compl[pnd_snd_idx]);
110 		memcpy(&pnd_snd, &link->wr_tx_pends[pnd_snd_idx],
111 		       sizeof(pnd_snd));
112 		/* clear the full struct smc_wr_tx_pend including .priv */
113 		memset(&link->wr_tx_pends[pnd_snd_idx], 0,
114 		       sizeof(link->wr_tx_pends[pnd_snd_idx]));
115 		memset(&link->wr_tx_bufs[pnd_snd_idx], 0,
116 		       sizeof(link->wr_tx_bufs[pnd_snd_idx]));
117 		if (!test_and_clear_bit(pnd_snd_idx, link->wr_tx_mask))
118 			return;
119 	}
120 
121 	if (wc->status) {
122 		if (link->lgr->smc_version == SMC_V2) {
123 			memset(link->wr_tx_v2_pend, 0,
124 			       sizeof(*link->wr_tx_v2_pend));
125 			memset(link->lgr->wr_tx_buf_v2, 0,
126 			       sizeof(*link->lgr->wr_tx_buf_v2));
127 		}
128 		/* terminate link */
129 		smcr_link_down_cond_sched(link);
130 	}
131 	if (pnd_snd.handler)
132 		pnd_snd.handler(&pnd_snd.priv, link, wc->status);
133 	wake_up(&link->wr_tx_wait);
134 }
135 
smc_wr_tx_tasklet_fn(struct tasklet_struct * t)136 static void smc_wr_tx_tasklet_fn(struct tasklet_struct *t)
137 {
138 	struct smc_ib_device *dev = from_tasklet(dev, t, send_tasklet);
139 	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
140 	int i = 0, rc;
141 	int polled = 0;
142 
143 again:
144 	polled++;
145 	do {
146 		memset(&wc, 0, sizeof(wc));
147 		rc = ib_poll_cq(dev->roce_cq_send, SMC_WR_MAX_POLL_CQE, wc);
148 		if (polled == 1) {
149 			ib_req_notify_cq(dev->roce_cq_send,
150 					 IB_CQ_NEXT_COMP |
151 					 IB_CQ_REPORT_MISSED_EVENTS);
152 		}
153 		if (!rc)
154 			break;
155 		for (i = 0; i < rc; i++)
156 			smc_wr_tx_process_cqe(&wc[i]);
157 	} while (rc > 0);
158 	if (polled == 1)
159 		goto again;
160 }
161 
smc_wr_tx_cq_handler(struct ib_cq * ib_cq,void * cq_context)162 void smc_wr_tx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
163 {
164 	struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
165 
166 	tasklet_schedule(&dev->send_tasklet);
167 }
168 
169 /*---------------------------- request submission ---------------------------*/
170 
smc_wr_tx_get_free_slot_index(struct smc_link * link,u32 * idx)171 static inline int smc_wr_tx_get_free_slot_index(struct smc_link *link, u32 *idx)
172 {
173 	*idx = link->wr_tx_cnt;
174 	if (!smc_link_sendable(link))
175 		return -ENOLINK;
176 	for_each_clear_bit(*idx, link->wr_tx_mask, link->wr_tx_cnt) {
177 		if (!test_and_set_bit(*idx, link->wr_tx_mask))
178 			return 0;
179 	}
180 	*idx = link->wr_tx_cnt;
181 	return -EBUSY;
182 }
183 
184 /**
185  * smc_wr_tx_get_free_slot() - returns buffer for message assembly,
186  *			and sets info for pending transmit tracking
187  * @link:		Pointer to smc_link used to later send the message.
188  * @handler:		Send completion handler function pointer.
189  * @wr_buf:		Out value returns pointer to message buffer.
190  * @wr_rdma_buf:	Out value returns pointer to rdma work request.
191  * @wr_pend_priv:	Out value returns pointer serving as handler context.
192  *
193  * Return: 0 on success, or -errno on error.
194  */
smc_wr_tx_get_free_slot(struct smc_link * link,smc_wr_tx_handler handler,struct smc_wr_buf ** wr_buf,struct smc_rdma_wr ** wr_rdma_buf,struct smc_wr_tx_pend_priv ** wr_pend_priv)195 int smc_wr_tx_get_free_slot(struct smc_link *link,
196 			    smc_wr_tx_handler handler,
197 			    struct smc_wr_buf **wr_buf,
198 			    struct smc_rdma_wr **wr_rdma_buf,
199 			    struct smc_wr_tx_pend_priv **wr_pend_priv)
200 {
201 	struct smc_link_group *lgr = smc_get_lgr(link);
202 	struct smc_wr_tx_pend *wr_pend;
203 	u32 idx = link->wr_tx_cnt;
204 	struct ib_send_wr *wr_ib;
205 	u64 wr_id;
206 	int rc;
207 
208 	*wr_buf = NULL;
209 	*wr_pend_priv = NULL;
210 	if (in_softirq() || lgr->terminating) {
211 		rc = smc_wr_tx_get_free_slot_index(link, &idx);
212 		if (rc)
213 			return rc;
214 	} else {
215 		rc = wait_event_interruptible_timeout(
216 			link->wr_tx_wait,
217 			!smc_link_sendable(link) ||
218 			lgr->terminating ||
219 			(smc_wr_tx_get_free_slot_index(link, &idx) != -EBUSY),
220 			SMC_WR_TX_WAIT_FREE_SLOT_TIME);
221 		if (!rc) {
222 			/* timeout - terminate link */
223 			smcr_link_down_cond_sched(link);
224 			return -EPIPE;
225 		}
226 		if (idx == link->wr_tx_cnt)
227 			return -EPIPE;
228 	}
229 	wr_id = smc_wr_tx_get_next_wr_id(link);
230 	wr_pend = &link->wr_tx_pends[idx];
231 	wr_pend->wr_id = wr_id;
232 	wr_pend->handler = handler;
233 	wr_pend->link = link;
234 	wr_pend->idx = idx;
235 	wr_ib = &link->wr_tx_ibs[idx];
236 	wr_ib->wr_id = wr_id;
237 	*wr_buf = &link->wr_tx_bufs[idx];
238 	if (wr_rdma_buf)
239 		*wr_rdma_buf = &link->wr_tx_rdmas[idx];
240 	*wr_pend_priv = &wr_pend->priv;
241 	return 0;
242 }
243 
smc_wr_tx_get_v2_slot(struct smc_link * link,smc_wr_tx_handler handler,struct smc_wr_v2_buf ** wr_buf,struct smc_wr_tx_pend_priv ** wr_pend_priv)244 int smc_wr_tx_get_v2_slot(struct smc_link *link,
245 			  smc_wr_tx_handler handler,
246 			  struct smc_wr_v2_buf **wr_buf,
247 			  struct smc_wr_tx_pend_priv **wr_pend_priv)
248 {
249 	struct smc_wr_tx_pend *wr_pend;
250 	struct ib_send_wr *wr_ib;
251 	u64 wr_id;
252 
253 	if (link->wr_tx_v2_pend->idx == link->wr_tx_cnt)
254 		return -EBUSY;
255 
256 	*wr_buf = NULL;
257 	*wr_pend_priv = NULL;
258 	wr_id = smc_wr_tx_get_next_wr_id(link);
259 	wr_pend = link->wr_tx_v2_pend;
260 	wr_pend->wr_id = wr_id;
261 	wr_pend->handler = handler;
262 	wr_pend->link = link;
263 	wr_pend->idx = link->wr_tx_cnt;
264 	wr_ib = link->wr_tx_v2_ib;
265 	wr_ib->wr_id = wr_id;
266 	*wr_buf = link->lgr->wr_tx_buf_v2;
267 	*wr_pend_priv = &wr_pend->priv;
268 	return 0;
269 }
270 
smc_wr_tx_put_slot(struct smc_link * link,struct smc_wr_tx_pend_priv * wr_pend_priv)271 int smc_wr_tx_put_slot(struct smc_link *link,
272 		       struct smc_wr_tx_pend_priv *wr_pend_priv)
273 {
274 	struct smc_wr_tx_pend *pend;
275 
276 	pend = container_of(wr_pend_priv, struct smc_wr_tx_pend, priv);
277 	if (pend->idx < link->wr_tx_cnt) {
278 		u32 idx = pend->idx;
279 
280 		/* clear the full struct smc_wr_tx_pend including .priv */
281 		memset(&link->wr_tx_pends[idx], 0,
282 		       sizeof(link->wr_tx_pends[idx]));
283 		memset(&link->wr_tx_bufs[idx], 0,
284 		       sizeof(link->wr_tx_bufs[idx]));
285 		test_and_clear_bit(idx, link->wr_tx_mask);
286 		wake_up(&link->wr_tx_wait);
287 		return 1;
288 	} else if (link->lgr->smc_version == SMC_V2 &&
289 		   pend->idx == link->wr_tx_cnt) {
290 		/* Large v2 buffer */
291 		memset(&link->wr_tx_v2_pend, 0,
292 		       sizeof(link->wr_tx_v2_pend));
293 		memset(&link->lgr->wr_tx_buf_v2, 0,
294 		       sizeof(link->lgr->wr_tx_buf_v2));
295 		return 1;
296 	}
297 
298 	return 0;
299 }
300 
301 /* Send prepared WR slot via ib_post_send.
302  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
303  */
smc_wr_tx_send(struct smc_link * link,struct smc_wr_tx_pend_priv * priv)304 int smc_wr_tx_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv)
305 {
306 	struct smc_wr_tx_pend *pend;
307 	int rc;
308 
309 	ib_req_notify_cq(link->smcibdev->roce_cq_send,
310 			 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
311 	pend = container_of(priv, struct smc_wr_tx_pend, priv);
312 	rc = ib_post_send(link->roce_qp, &link->wr_tx_ibs[pend->idx], NULL);
313 	if (rc) {
314 		smc_wr_tx_put_slot(link, priv);
315 		smcr_link_down_cond_sched(link);
316 	}
317 	return rc;
318 }
319 
smc_wr_tx_v2_send(struct smc_link * link,struct smc_wr_tx_pend_priv * priv,int len)320 int smc_wr_tx_v2_send(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
321 		      int len)
322 {
323 	int rc;
324 
325 	link->wr_tx_v2_ib->sg_list[0].length = len;
326 	ib_req_notify_cq(link->smcibdev->roce_cq_send,
327 			 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
328 	rc = ib_post_send(link->roce_qp, link->wr_tx_v2_ib, NULL);
329 	if (rc) {
330 		smc_wr_tx_put_slot(link, priv);
331 		smcr_link_down_cond_sched(link);
332 	}
333 	return rc;
334 }
335 
336 /* Send prepared WR slot via ib_post_send and wait for send completion
337  * notification.
338  * @priv: pointer to smc_wr_tx_pend_priv identifying prepared message buffer
339  */
smc_wr_tx_send_wait(struct smc_link * link,struct smc_wr_tx_pend_priv * priv,unsigned long timeout)340 int smc_wr_tx_send_wait(struct smc_link *link, struct smc_wr_tx_pend_priv *priv,
341 			unsigned long timeout)
342 {
343 	struct smc_wr_tx_pend *pend;
344 	u32 pnd_idx;
345 	int rc;
346 
347 	pend = container_of(priv, struct smc_wr_tx_pend, priv);
348 	pend->compl_requested = 1;
349 	pnd_idx = pend->idx;
350 	init_completion(&link->wr_tx_compl[pnd_idx]);
351 
352 	rc = smc_wr_tx_send(link, priv);
353 	if (rc)
354 		return rc;
355 	/* wait for completion by smc_wr_tx_process_cqe() */
356 	rc = wait_for_completion_interruptible_timeout(
357 					&link->wr_tx_compl[pnd_idx], timeout);
358 	if (rc <= 0)
359 		rc = -ENODATA;
360 	if (rc > 0)
361 		rc = 0;
362 	return rc;
363 }
364 
365 /* Register a memory region and wait for result. */
smc_wr_reg_send(struct smc_link * link,struct ib_mr * mr)366 int smc_wr_reg_send(struct smc_link *link, struct ib_mr *mr)
367 {
368 	int rc;
369 
370 	ib_req_notify_cq(link->smcibdev->roce_cq_send,
371 			 IB_CQ_NEXT_COMP | IB_CQ_REPORT_MISSED_EVENTS);
372 	link->wr_reg_state = POSTED;
373 	link->wr_reg.wr.wr_id = (u64)(uintptr_t)mr;
374 	link->wr_reg.mr = mr;
375 	link->wr_reg.key = mr->rkey;
376 	rc = ib_post_send(link->roce_qp, &link->wr_reg.wr, NULL);
377 	if (rc)
378 		return rc;
379 
380 	atomic_inc(&link->wr_reg_refcnt);
381 	rc = wait_event_interruptible_timeout(link->wr_reg_wait,
382 					      (link->wr_reg_state != POSTED),
383 					      SMC_WR_REG_MR_WAIT_TIME);
384 	if (atomic_dec_and_test(&link->wr_reg_refcnt))
385 		wake_up_all(&link->wr_reg_wait);
386 	if (!rc) {
387 		/* timeout - terminate link */
388 		smcr_link_down_cond_sched(link);
389 		return -EPIPE;
390 	}
391 	if (rc == -ERESTARTSYS)
392 		return -EINTR;
393 	switch (link->wr_reg_state) {
394 	case CONFIRMED:
395 		rc = 0;
396 		break;
397 	case FAILED:
398 		rc = -EIO;
399 		break;
400 	case POSTED:
401 		rc = -EPIPE;
402 		break;
403 	}
404 	return rc;
405 }
406 
407 /****************************** receive queue ********************************/
408 
smc_wr_rx_register_handler(struct smc_wr_rx_handler * handler)409 int smc_wr_rx_register_handler(struct smc_wr_rx_handler *handler)
410 {
411 	struct smc_wr_rx_handler *h_iter;
412 	int rc = 0;
413 
414 	spin_lock(&smc_wr_rx_hash_lock);
415 	hash_for_each_possible(smc_wr_rx_hash, h_iter, list, handler->type) {
416 		if (h_iter->type == handler->type) {
417 			rc = -EEXIST;
418 			goto out_unlock;
419 		}
420 	}
421 	hash_add(smc_wr_rx_hash, &handler->list, handler->type);
422 out_unlock:
423 	spin_unlock(&smc_wr_rx_hash_lock);
424 	return rc;
425 }
426 
427 /* Demultiplex a received work request based on the message type to its handler.
428  * Relies on smc_wr_rx_hash having been completely filled before any IB WRs,
429  * and not being modified any more afterwards so we don't need to lock it.
430  */
smc_wr_rx_demultiplex(struct ib_wc * wc)431 static inline void smc_wr_rx_demultiplex(struct ib_wc *wc)
432 {
433 	struct smc_link *link = (struct smc_link *)wc->qp->qp_context;
434 	struct smc_wr_rx_handler *handler;
435 	struct smc_wr_rx_hdr *wr_rx;
436 	u64 temp_wr_id;
437 	u32 index;
438 
439 	if (wc->byte_len < sizeof(*wr_rx))
440 		return; /* short message */
441 	temp_wr_id = wc->wr_id;
442 	index = do_div(temp_wr_id, link->wr_rx_cnt);
443 	wr_rx = (struct smc_wr_rx_hdr *)&link->wr_rx_bufs[index];
444 	hash_for_each_possible(smc_wr_rx_hash, handler, list, wr_rx->type) {
445 		if (handler->type == wr_rx->type)
446 			handler->handler(wc, wr_rx);
447 	}
448 }
449 
smc_wr_rx_process_cqes(struct ib_wc wc[],int num)450 static inline void smc_wr_rx_process_cqes(struct ib_wc wc[], int num)
451 {
452 	struct smc_link *link;
453 	int i;
454 
455 	for (i = 0; i < num; i++) {
456 		link = wc[i].qp->qp_context;
457 		link->wr_rx_id_compl = wc[i].wr_id;
458 		if (wc[i].status == IB_WC_SUCCESS) {
459 			link->wr_rx_tstamp = jiffies;
460 			smc_wr_rx_demultiplex(&wc[i]);
461 			smc_wr_rx_post(link); /* refill WR RX */
462 		} else {
463 			/* handle status errors */
464 			switch (wc[i].status) {
465 			case IB_WC_RETRY_EXC_ERR:
466 			case IB_WC_RNR_RETRY_EXC_ERR:
467 			case IB_WC_WR_FLUSH_ERR:
468 				smcr_link_down_cond_sched(link);
469 				if (link->wr_rx_id_compl == link->wr_rx_id)
470 					wake_up(&link->wr_rx_empty_wait);
471 				break;
472 			default:
473 				smc_wr_rx_post(link); /* refill WR RX */
474 				break;
475 			}
476 		}
477 	}
478 }
479 
smc_wr_rx_tasklet_fn(struct tasklet_struct * t)480 static void smc_wr_rx_tasklet_fn(struct tasklet_struct *t)
481 {
482 	struct smc_ib_device *dev = from_tasklet(dev, t, recv_tasklet);
483 	struct ib_wc wc[SMC_WR_MAX_POLL_CQE];
484 	int polled = 0;
485 	int rc;
486 
487 again:
488 	polled++;
489 	do {
490 		memset(&wc, 0, sizeof(wc));
491 		rc = ib_poll_cq(dev->roce_cq_recv, SMC_WR_MAX_POLL_CQE, wc);
492 		if (polled == 1) {
493 			ib_req_notify_cq(dev->roce_cq_recv,
494 					 IB_CQ_SOLICITED_MASK
495 					 | IB_CQ_REPORT_MISSED_EVENTS);
496 		}
497 		if (!rc)
498 			break;
499 		smc_wr_rx_process_cqes(&wc[0], rc);
500 	} while (rc > 0);
501 	if (polled == 1)
502 		goto again;
503 }
504 
smc_wr_rx_cq_handler(struct ib_cq * ib_cq,void * cq_context)505 void smc_wr_rx_cq_handler(struct ib_cq *ib_cq, void *cq_context)
506 {
507 	struct smc_ib_device *dev = (struct smc_ib_device *)cq_context;
508 
509 	tasklet_schedule(&dev->recv_tasklet);
510 }
511 
smc_wr_rx_post_init(struct smc_link * link)512 int smc_wr_rx_post_init(struct smc_link *link)
513 {
514 	u32 i;
515 	int rc = 0;
516 
517 	for (i = 0; i < link->wr_rx_cnt; i++)
518 		rc = smc_wr_rx_post(link);
519 	return rc;
520 }
521 
522 /***************************** init, exit, misc ******************************/
523 
smc_wr_remember_qp_attr(struct smc_link * lnk)524 void smc_wr_remember_qp_attr(struct smc_link *lnk)
525 {
526 	struct ib_qp_attr *attr = &lnk->qp_attr;
527 	struct ib_qp_init_attr init_attr;
528 
529 	memset(attr, 0, sizeof(*attr));
530 	memset(&init_attr, 0, sizeof(init_attr));
531 	ib_query_qp(lnk->roce_qp, attr,
532 		    IB_QP_STATE |
533 		    IB_QP_CUR_STATE |
534 		    IB_QP_PKEY_INDEX |
535 		    IB_QP_PORT |
536 		    IB_QP_QKEY |
537 		    IB_QP_AV |
538 		    IB_QP_PATH_MTU |
539 		    IB_QP_TIMEOUT |
540 		    IB_QP_RETRY_CNT |
541 		    IB_QP_RNR_RETRY |
542 		    IB_QP_RQ_PSN |
543 		    IB_QP_ALT_PATH |
544 		    IB_QP_MIN_RNR_TIMER |
545 		    IB_QP_SQ_PSN |
546 		    IB_QP_PATH_MIG_STATE |
547 		    IB_QP_CAP |
548 		    IB_QP_DEST_QPN,
549 		    &init_attr);
550 
551 	lnk->wr_tx_cnt = min_t(size_t, SMC_WR_BUF_CNT,
552 			       lnk->qp_attr.cap.max_send_wr);
553 	lnk->wr_rx_cnt = min_t(size_t, SMC_WR_BUF_CNT * 3,
554 			       lnk->qp_attr.cap.max_recv_wr);
555 }
556 
smc_wr_init_sge(struct smc_link * lnk)557 static void smc_wr_init_sge(struct smc_link *lnk)
558 {
559 	int sges_per_buf = (lnk->lgr->smc_version == SMC_V2) ? 2 : 1;
560 	bool send_inline = (lnk->qp_attr.cap.max_inline_data > SMC_WR_TX_SIZE);
561 	u32 i;
562 
563 	for (i = 0; i < lnk->wr_tx_cnt; i++) {
564 		lnk->wr_tx_sges[i].addr = send_inline ? (uintptr_t)(&lnk->wr_tx_bufs[i]) :
565 			lnk->wr_tx_dma_addr + i * SMC_WR_BUF_SIZE;
566 		lnk->wr_tx_sges[i].length = SMC_WR_TX_SIZE;
567 		lnk->wr_tx_sges[i].lkey = lnk->roce_pd->local_dma_lkey;
568 		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[0].lkey =
569 			lnk->roce_pd->local_dma_lkey;
570 		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge[1].lkey =
571 			lnk->roce_pd->local_dma_lkey;
572 		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[0].lkey =
573 			lnk->roce_pd->local_dma_lkey;
574 		lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge[1].lkey =
575 			lnk->roce_pd->local_dma_lkey;
576 		lnk->wr_tx_ibs[i].next = NULL;
577 		lnk->wr_tx_ibs[i].sg_list = &lnk->wr_tx_sges[i];
578 		lnk->wr_tx_ibs[i].num_sge = 1;
579 		lnk->wr_tx_ibs[i].opcode = IB_WR_SEND;
580 		lnk->wr_tx_ibs[i].send_flags =
581 			IB_SEND_SIGNALED | IB_SEND_SOLICITED;
582 		if (send_inline)
583 			lnk->wr_tx_ibs[i].send_flags |= IB_SEND_INLINE;
584 		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.opcode = IB_WR_RDMA_WRITE;
585 		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.opcode = IB_WR_RDMA_WRITE;
586 		lnk->wr_tx_rdmas[i].wr_tx_rdma[0].wr.sg_list =
587 			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[0].wr_tx_rdma_sge;
588 		lnk->wr_tx_rdmas[i].wr_tx_rdma[1].wr.sg_list =
589 			lnk->wr_tx_rdma_sges[i].tx_rdma_sge[1].wr_tx_rdma_sge;
590 	}
591 
592 	if (lnk->lgr->smc_version == SMC_V2) {
593 		lnk->wr_tx_v2_sge->addr = lnk->wr_tx_v2_dma_addr;
594 		lnk->wr_tx_v2_sge->length = SMC_WR_BUF_V2_SIZE;
595 		lnk->wr_tx_v2_sge->lkey = lnk->roce_pd->local_dma_lkey;
596 
597 		lnk->wr_tx_v2_ib->next = NULL;
598 		lnk->wr_tx_v2_ib->sg_list = lnk->wr_tx_v2_sge;
599 		lnk->wr_tx_v2_ib->num_sge = 1;
600 		lnk->wr_tx_v2_ib->opcode = IB_WR_SEND;
601 		lnk->wr_tx_v2_ib->send_flags =
602 			IB_SEND_SIGNALED | IB_SEND_SOLICITED;
603 	}
604 
605 	/* With SMC-Rv2 there can be messages larger than SMC_WR_TX_SIZE.
606 	 * Each ib_recv_wr gets 2 sges, the second one is a spillover buffer
607 	 * and the same buffer for all sges. When a larger message arrived then
608 	 * the content of the first small sge is copied to the beginning of
609 	 * the larger spillover buffer, allowing easy data mapping.
610 	 */
611 	for (i = 0; i < lnk->wr_rx_cnt; i++) {
612 		int x = i * sges_per_buf;
613 
614 		lnk->wr_rx_sges[x].addr =
615 			lnk->wr_rx_dma_addr + i * SMC_WR_BUF_SIZE;
616 		lnk->wr_rx_sges[x].length = SMC_WR_TX_SIZE;
617 		lnk->wr_rx_sges[x].lkey = lnk->roce_pd->local_dma_lkey;
618 		if (lnk->lgr->smc_version == SMC_V2) {
619 			lnk->wr_rx_sges[x + 1].addr =
620 					lnk->wr_rx_v2_dma_addr + SMC_WR_TX_SIZE;
621 			lnk->wr_rx_sges[x + 1].length =
622 					SMC_WR_BUF_V2_SIZE - SMC_WR_TX_SIZE;
623 			lnk->wr_rx_sges[x + 1].lkey =
624 					lnk->roce_pd->local_dma_lkey;
625 		}
626 		lnk->wr_rx_ibs[i].next = NULL;
627 		lnk->wr_rx_ibs[i].sg_list = &lnk->wr_rx_sges[x];
628 		lnk->wr_rx_ibs[i].num_sge = sges_per_buf;
629 	}
630 	lnk->wr_reg.wr.next = NULL;
631 	lnk->wr_reg.wr.num_sge = 0;
632 	lnk->wr_reg.wr.send_flags = IB_SEND_SIGNALED;
633 	lnk->wr_reg.wr.opcode = IB_WR_REG_MR;
634 	lnk->wr_reg.access = IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE;
635 }
636 
smc_wr_free_link(struct smc_link * lnk)637 void smc_wr_free_link(struct smc_link *lnk)
638 {
639 	struct ib_device *ibdev;
640 
641 	if (!lnk->smcibdev)
642 		return;
643 	ibdev = lnk->smcibdev->ibdev;
644 
645 	smc_wr_drain_cq(lnk);
646 	smc_wr_wakeup_reg_wait(lnk);
647 	smc_wr_wakeup_tx_wait(lnk);
648 
649 	smc_wr_tx_wait_no_pending_sends(lnk);
650 	wait_event(lnk->wr_reg_wait, (!atomic_read(&lnk->wr_reg_refcnt)));
651 	wait_event(lnk->wr_tx_wait, (!atomic_read(&lnk->wr_tx_refcnt)));
652 
653 	if (lnk->wr_rx_dma_addr) {
654 		ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
655 				    SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
656 				    DMA_FROM_DEVICE);
657 		lnk->wr_rx_dma_addr = 0;
658 	}
659 	if (lnk->wr_rx_v2_dma_addr) {
660 		ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr,
661 				    SMC_WR_BUF_V2_SIZE,
662 				    DMA_FROM_DEVICE);
663 		lnk->wr_rx_v2_dma_addr = 0;
664 	}
665 	if (lnk->wr_tx_dma_addr) {
666 		ib_dma_unmap_single(ibdev, lnk->wr_tx_dma_addr,
667 				    SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
668 				    DMA_TO_DEVICE);
669 		lnk->wr_tx_dma_addr = 0;
670 	}
671 	if (lnk->wr_tx_v2_dma_addr) {
672 		ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr,
673 				    SMC_WR_BUF_V2_SIZE,
674 				    DMA_TO_DEVICE);
675 		lnk->wr_tx_v2_dma_addr = 0;
676 	}
677 }
678 
smc_wr_free_lgr_mem(struct smc_link_group * lgr)679 void smc_wr_free_lgr_mem(struct smc_link_group *lgr)
680 {
681 	if (lgr->smc_version < SMC_V2)
682 		return;
683 
684 	kfree(lgr->wr_rx_buf_v2);
685 	lgr->wr_rx_buf_v2 = NULL;
686 	kfree(lgr->wr_tx_buf_v2);
687 	lgr->wr_tx_buf_v2 = NULL;
688 }
689 
smc_wr_free_link_mem(struct smc_link * lnk)690 void smc_wr_free_link_mem(struct smc_link *lnk)
691 {
692 	kfree(lnk->wr_tx_v2_ib);
693 	lnk->wr_tx_v2_ib = NULL;
694 	kfree(lnk->wr_tx_v2_sge);
695 	lnk->wr_tx_v2_sge = NULL;
696 	kfree(lnk->wr_tx_v2_pend);
697 	lnk->wr_tx_v2_pend = NULL;
698 	kfree(lnk->wr_tx_compl);
699 	lnk->wr_tx_compl = NULL;
700 	kfree(lnk->wr_tx_pends);
701 	lnk->wr_tx_pends = NULL;
702 	bitmap_free(lnk->wr_tx_mask);
703 	lnk->wr_tx_mask = NULL;
704 	kfree(lnk->wr_tx_sges);
705 	lnk->wr_tx_sges = NULL;
706 	kfree(lnk->wr_tx_rdma_sges);
707 	lnk->wr_tx_rdma_sges = NULL;
708 	kfree(lnk->wr_rx_sges);
709 	lnk->wr_rx_sges = NULL;
710 	kfree(lnk->wr_tx_rdmas);
711 	lnk->wr_tx_rdmas = NULL;
712 	kfree(lnk->wr_rx_ibs);
713 	lnk->wr_rx_ibs = NULL;
714 	kfree(lnk->wr_tx_ibs);
715 	lnk->wr_tx_ibs = NULL;
716 	kfree(lnk->wr_tx_bufs);
717 	lnk->wr_tx_bufs = NULL;
718 	kfree(lnk->wr_rx_bufs);
719 	lnk->wr_rx_bufs = NULL;
720 }
721 
smc_wr_alloc_lgr_mem(struct smc_link_group * lgr)722 int smc_wr_alloc_lgr_mem(struct smc_link_group *lgr)
723 {
724 	if (lgr->smc_version < SMC_V2)
725 		return 0;
726 
727 	lgr->wr_rx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL);
728 	if (!lgr->wr_rx_buf_v2)
729 		return -ENOMEM;
730 	lgr->wr_tx_buf_v2 = kzalloc(SMC_WR_BUF_V2_SIZE, GFP_KERNEL);
731 	if (!lgr->wr_tx_buf_v2) {
732 		kfree(lgr->wr_rx_buf_v2);
733 		return -ENOMEM;
734 	}
735 	return 0;
736 }
737 
smc_wr_alloc_link_mem(struct smc_link * link)738 int smc_wr_alloc_link_mem(struct smc_link *link)
739 {
740 	int sges_per_buf = link->lgr->smc_version == SMC_V2 ? 2 : 1;
741 
742 	/* allocate link related memory */
743 	link->wr_tx_bufs = kcalloc(SMC_WR_BUF_CNT, SMC_WR_BUF_SIZE, GFP_KERNEL);
744 	if (!link->wr_tx_bufs)
745 		goto no_mem;
746 	link->wr_rx_bufs = kcalloc(SMC_WR_BUF_CNT * 3, SMC_WR_BUF_SIZE,
747 				   GFP_KERNEL);
748 	if (!link->wr_rx_bufs)
749 		goto no_mem_wr_tx_bufs;
750 	link->wr_tx_ibs = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_ibs[0]),
751 				  GFP_KERNEL);
752 	if (!link->wr_tx_ibs)
753 		goto no_mem_wr_rx_bufs;
754 	link->wr_rx_ibs = kcalloc(SMC_WR_BUF_CNT * 3,
755 				  sizeof(link->wr_rx_ibs[0]),
756 				  GFP_KERNEL);
757 	if (!link->wr_rx_ibs)
758 		goto no_mem_wr_tx_ibs;
759 	link->wr_tx_rdmas = kcalloc(SMC_WR_BUF_CNT,
760 				    sizeof(link->wr_tx_rdmas[0]),
761 				    GFP_KERNEL);
762 	if (!link->wr_tx_rdmas)
763 		goto no_mem_wr_rx_ibs;
764 	link->wr_tx_rdma_sges = kcalloc(SMC_WR_BUF_CNT,
765 					sizeof(link->wr_tx_rdma_sges[0]),
766 					GFP_KERNEL);
767 	if (!link->wr_tx_rdma_sges)
768 		goto no_mem_wr_tx_rdmas;
769 	link->wr_tx_sges = kcalloc(SMC_WR_BUF_CNT, sizeof(link->wr_tx_sges[0]),
770 				   GFP_KERNEL);
771 	if (!link->wr_tx_sges)
772 		goto no_mem_wr_tx_rdma_sges;
773 	link->wr_rx_sges = kcalloc(SMC_WR_BUF_CNT * 3,
774 				   sizeof(link->wr_rx_sges[0]) * sges_per_buf,
775 				   GFP_KERNEL);
776 	if (!link->wr_rx_sges)
777 		goto no_mem_wr_tx_sges;
778 	link->wr_tx_mask = bitmap_zalloc(SMC_WR_BUF_CNT, GFP_KERNEL);
779 	if (!link->wr_tx_mask)
780 		goto no_mem_wr_rx_sges;
781 	link->wr_tx_pends = kcalloc(SMC_WR_BUF_CNT,
782 				    sizeof(link->wr_tx_pends[0]),
783 				    GFP_KERNEL);
784 	if (!link->wr_tx_pends)
785 		goto no_mem_wr_tx_mask;
786 	link->wr_tx_compl = kcalloc(SMC_WR_BUF_CNT,
787 				    sizeof(link->wr_tx_compl[0]),
788 				    GFP_KERNEL);
789 	if (!link->wr_tx_compl)
790 		goto no_mem_wr_tx_pends;
791 
792 	if (link->lgr->smc_version == SMC_V2) {
793 		link->wr_tx_v2_ib = kzalloc(sizeof(*link->wr_tx_v2_ib),
794 					    GFP_KERNEL);
795 		if (!link->wr_tx_v2_ib)
796 			goto no_mem_tx_compl;
797 		link->wr_tx_v2_sge = kzalloc(sizeof(*link->wr_tx_v2_sge),
798 					     GFP_KERNEL);
799 		if (!link->wr_tx_v2_sge)
800 			goto no_mem_v2_ib;
801 		link->wr_tx_v2_pend = kzalloc(sizeof(*link->wr_tx_v2_pend),
802 					      GFP_KERNEL);
803 		if (!link->wr_tx_v2_pend)
804 			goto no_mem_v2_sge;
805 	}
806 	return 0;
807 
808 no_mem_v2_sge:
809 	kfree(link->wr_tx_v2_sge);
810 no_mem_v2_ib:
811 	kfree(link->wr_tx_v2_ib);
812 no_mem_tx_compl:
813 	kfree(link->wr_tx_compl);
814 no_mem_wr_tx_pends:
815 	kfree(link->wr_tx_pends);
816 no_mem_wr_tx_mask:
817 	kfree(link->wr_tx_mask);
818 no_mem_wr_rx_sges:
819 	kfree(link->wr_rx_sges);
820 no_mem_wr_tx_sges:
821 	kfree(link->wr_tx_sges);
822 no_mem_wr_tx_rdma_sges:
823 	kfree(link->wr_tx_rdma_sges);
824 no_mem_wr_tx_rdmas:
825 	kfree(link->wr_tx_rdmas);
826 no_mem_wr_rx_ibs:
827 	kfree(link->wr_rx_ibs);
828 no_mem_wr_tx_ibs:
829 	kfree(link->wr_tx_ibs);
830 no_mem_wr_rx_bufs:
831 	kfree(link->wr_rx_bufs);
832 no_mem_wr_tx_bufs:
833 	kfree(link->wr_tx_bufs);
834 no_mem:
835 	return -ENOMEM;
836 }
837 
smc_wr_remove_dev(struct smc_ib_device * smcibdev)838 void smc_wr_remove_dev(struct smc_ib_device *smcibdev)
839 {
840 	tasklet_kill(&smcibdev->recv_tasklet);
841 	tasklet_kill(&smcibdev->send_tasklet);
842 }
843 
smc_wr_add_dev(struct smc_ib_device * smcibdev)844 void smc_wr_add_dev(struct smc_ib_device *smcibdev)
845 {
846 	tasklet_setup(&smcibdev->recv_tasklet, smc_wr_rx_tasklet_fn);
847 	tasklet_setup(&smcibdev->send_tasklet, smc_wr_tx_tasklet_fn);
848 }
849 
smc_wr_create_link(struct smc_link * lnk)850 int smc_wr_create_link(struct smc_link *lnk)
851 {
852 	struct ib_device *ibdev = lnk->smcibdev->ibdev;
853 	int rc = 0;
854 
855 	smc_wr_tx_set_wr_id(&lnk->wr_tx_id, 0);
856 	lnk->wr_rx_id = 0;
857 	lnk->wr_rx_dma_addr = ib_dma_map_single(
858 		ibdev, lnk->wr_rx_bufs,	SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
859 		DMA_FROM_DEVICE);
860 	if (ib_dma_mapping_error(ibdev, lnk->wr_rx_dma_addr)) {
861 		lnk->wr_rx_dma_addr = 0;
862 		rc = -EIO;
863 		goto out;
864 	}
865 	if (lnk->lgr->smc_version == SMC_V2) {
866 		lnk->wr_rx_v2_dma_addr = ib_dma_map_single(ibdev,
867 			lnk->lgr->wr_rx_buf_v2, SMC_WR_BUF_V2_SIZE,
868 			DMA_FROM_DEVICE);
869 		if (ib_dma_mapping_error(ibdev, lnk->wr_rx_v2_dma_addr)) {
870 			lnk->wr_rx_v2_dma_addr = 0;
871 			rc = -EIO;
872 			goto dma_unmap;
873 		}
874 		lnk->wr_tx_v2_dma_addr = ib_dma_map_single(ibdev,
875 			lnk->lgr->wr_tx_buf_v2, SMC_WR_BUF_V2_SIZE,
876 			DMA_TO_DEVICE);
877 		if (ib_dma_mapping_error(ibdev, lnk->wr_tx_v2_dma_addr)) {
878 			lnk->wr_tx_v2_dma_addr = 0;
879 			rc = -EIO;
880 			goto dma_unmap;
881 		}
882 	}
883 	lnk->wr_tx_dma_addr = ib_dma_map_single(
884 		ibdev, lnk->wr_tx_bufs,	SMC_WR_BUF_SIZE * lnk->wr_tx_cnt,
885 		DMA_TO_DEVICE);
886 	if (ib_dma_mapping_error(ibdev, lnk->wr_tx_dma_addr)) {
887 		rc = -EIO;
888 		goto dma_unmap;
889 	}
890 	smc_wr_init_sge(lnk);
891 	bitmap_zero(lnk->wr_tx_mask, SMC_WR_BUF_CNT);
892 	init_waitqueue_head(&lnk->wr_tx_wait);
893 	atomic_set(&lnk->wr_tx_refcnt, 0);
894 	init_waitqueue_head(&lnk->wr_reg_wait);
895 	atomic_set(&lnk->wr_reg_refcnt, 0);
896 	init_waitqueue_head(&lnk->wr_rx_empty_wait);
897 	return rc;
898 
899 dma_unmap:
900 	if (lnk->wr_rx_v2_dma_addr) {
901 		ib_dma_unmap_single(ibdev, lnk->wr_rx_v2_dma_addr,
902 				    SMC_WR_BUF_V2_SIZE,
903 				    DMA_FROM_DEVICE);
904 		lnk->wr_rx_v2_dma_addr = 0;
905 	}
906 	if (lnk->wr_tx_v2_dma_addr) {
907 		ib_dma_unmap_single(ibdev, lnk->wr_tx_v2_dma_addr,
908 				    SMC_WR_BUF_V2_SIZE,
909 				    DMA_TO_DEVICE);
910 		lnk->wr_tx_v2_dma_addr = 0;
911 	}
912 	ib_dma_unmap_single(ibdev, lnk->wr_rx_dma_addr,
913 			    SMC_WR_BUF_SIZE * lnk->wr_rx_cnt,
914 			    DMA_FROM_DEVICE);
915 	lnk->wr_rx_dma_addr = 0;
916 out:
917 	return rc;
918 }
919