1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "osdep.h"
4 #include "hmc.h"
5 #include "defs.h"
6 #include "type.h"
7 #include "protos.h"
8 #include "puda.h"
9 #include "ws.h"
10 
11 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
12 			      struct irdma_puda_buf *buf);
13 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid);
14 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
15 				     struct irdma_puda_buf *buf, u32 wqe_idx);
16 /**
17  * irdma_puda_get_listbuf - get buffer from puda list
18  * @list: list to use for buffers (ILQ or IEQ)
19  */
irdma_puda_get_listbuf(struct list_head * list)20 static struct irdma_puda_buf *irdma_puda_get_listbuf(struct list_head *list)
21 {
22 	struct irdma_puda_buf *buf = NULL;
23 
24 	if (!list_empty(list)) {
25 		buf = (struct irdma_puda_buf *)list->next;
26 		list_del((struct list_head *)&buf->list);
27 	}
28 
29 	return buf;
30 }
31 
32 /**
33  * irdma_puda_get_bufpool - return buffer from resource
34  * @rsrc: resource to use for buffer
35  */
irdma_puda_get_bufpool(struct irdma_puda_rsrc * rsrc)36 struct irdma_puda_buf *irdma_puda_get_bufpool(struct irdma_puda_rsrc *rsrc)
37 {
38 	struct irdma_puda_buf *buf = NULL;
39 	struct list_head *list = &rsrc->bufpool;
40 	unsigned long flags;
41 
42 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
43 	buf = irdma_puda_get_listbuf(list);
44 	if (buf) {
45 		rsrc->avail_buf_count--;
46 		buf->vsi = rsrc->vsi;
47 	} else {
48 		rsrc->stats_buf_alloc_fail++;
49 	}
50 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
51 
52 	return buf;
53 }
54 
55 /**
56  * irdma_puda_ret_bufpool - return buffer to rsrc list
57  * @rsrc: resource to use for buffer
58  * @buf: buffer to return to resource
59  */
irdma_puda_ret_bufpool(struct irdma_puda_rsrc * rsrc,struct irdma_puda_buf * buf)60 void irdma_puda_ret_bufpool(struct irdma_puda_rsrc *rsrc,
61 			    struct irdma_puda_buf *buf)
62 {
63 	unsigned long flags;
64 
65 	buf->do_lpb = false;
66 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
67 	list_add(&buf->list, &rsrc->bufpool);
68 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
69 	rsrc->avail_buf_count++;
70 }
71 
72 /**
73  * irdma_puda_post_recvbuf - set wqe for rcv buffer
74  * @rsrc: resource ptr
75  * @wqe_idx: wqe index to use
76  * @buf: puda buffer for rcv q
77  * @initial: flag if during init time
78  */
irdma_puda_post_recvbuf(struct irdma_puda_rsrc * rsrc,u32 wqe_idx,struct irdma_puda_buf * buf,bool initial)79 static void irdma_puda_post_recvbuf(struct irdma_puda_rsrc *rsrc, u32 wqe_idx,
80 				    struct irdma_puda_buf *buf, bool initial)
81 {
82 	__le64 *wqe;
83 	struct irdma_sc_qp *qp = &rsrc->qp;
84 	u64 offset24 = 0;
85 
86 	/* Synch buffer for use by device */
87 	dma_sync_single_for_device(rsrc->dev->hw->device, buf->mem.pa,
88 				   buf->mem.size, DMA_BIDIRECTIONAL);
89 	qp->qp_uk.rq_wrid_array[wqe_idx] = (uintptr_t)buf;
90 	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
91 	if (!initial)
92 		get_64bit_val(wqe, 24, &offset24);
93 
94 	offset24 = (offset24) ? 0 : FIELD_PREP(IRDMAQPSQ_VALID, 1);
95 
96 	set_64bit_val(wqe, 16, 0);
97 	set_64bit_val(wqe, 0, buf->mem.pa);
98 	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
99 		set_64bit_val(wqe, 8,
100 			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, buf->mem.size));
101 	} else {
102 		set_64bit_val(wqe, 8,
103 			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, buf->mem.size) |
104 			      offset24);
105 	}
106 	dma_wmb(); /* make sure WQE is written before valid bit is set */
107 
108 	set_64bit_val(wqe, 24, offset24);
109 }
110 
111 /**
112  * irdma_puda_replenish_rq - post rcv buffers
113  * @rsrc: resource to use for buffer
114  * @initial: flag if during init time
115  */
irdma_puda_replenish_rq(struct irdma_puda_rsrc * rsrc,bool initial)116 static int irdma_puda_replenish_rq(struct irdma_puda_rsrc *rsrc, bool initial)
117 {
118 	u32 i;
119 	u32 invalid_cnt = rsrc->rxq_invalid_cnt;
120 	struct irdma_puda_buf *buf = NULL;
121 
122 	for (i = 0; i < invalid_cnt; i++) {
123 		buf = irdma_puda_get_bufpool(rsrc);
124 		if (!buf)
125 			return -ENOBUFS;
126 		irdma_puda_post_recvbuf(rsrc, rsrc->rx_wqe_idx, buf, initial);
127 		rsrc->rx_wqe_idx = ((rsrc->rx_wqe_idx + 1) % rsrc->rq_size);
128 		rsrc->rxq_invalid_cnt--;
129 	}
130 
131 	return 0;
132 }
133 
134 /**
135  * irdma_puda_alloc_buf - allocate mem for buffer
136  * @dev: iwarp device
137  * @len: length of buffer
138  */
irdma_puda_alloc_buf(struct irdma_sc_dev * dev,u32 len)139 static struct irdma_puda_buf *irdma_puda_alloc_buf(struct irdma_sc_dev *dev,
140 						   u32 len)
141 {
142 	struct irdma_puda_buf *buf;
143 	struct irdma_virt_mem buf_mem;
144 
145 	buf_mem.size = sizeof(struct irdma_puda_buf);
146 	buf_mem.va = kzalloc(buf_mem.size, GFP_KERNEL);
147 	if (!buf_mem.va)
148 		return NULL;
149 
150 	buf = buf_mem.va;
151 	buf->mem.size = len;
152 	buf->mem.va = kzalloc(buf->mem.size, GFP_KERNEL);
153 	if (!buf->mem.va)
154 		goto free_virt;
155 	buf->mem.pa = dma_map_single(dev->hw->device, buf->mem.va,
156 				     buf->mem.size, DMA_BIDIRECTIONAL);
157 	if (dma_mapping_error(dev->hw->device, buf->mem.pa)) {
158 		kfree(buf->mem.va);
159 		goto free_virt;
160 	}
161 
162 	buf->buf_mem.va = buf_mem.va;
163 	buf->buf_mem.size = buf_mem.size;
164 
165 	return buf;
166 
167 free_virt:
168 	kfree(buf_mem.va);
169 	return NULL;
170 }
171 
172 /**
173  * irdma_puda_dele_buf - delete buffer back to system
174  * @dev: iwarp device
175  * @buf: buffer to free
176  */
irdma_puda_dele_buf(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)177 static void irdma_puda_dele_buf(struct irdma_sc_dev *dev,
178 				struct irdma_puda_buf *buf)
179 {
180 	dma_unmap_single(dev->hw->device, buf->mem.pa, buf->mem.size,
181 			 DMA_BIDIRECTIONAL);
182 	kfree(buf->mem.va);
183 	kfree(buf->buf_mem.va);
184 }
185 
186 /**
187  * irdma_puda_get_next_send_wqe - return next wqe for processing
188  * @qp: puda qp for wqe
189  * @wqe_idx: wqe index for caller
190  */
irdma_puda_get_next_send_wqe(struct irdma_qp_uk * qp,u32 * wqe_idx)191 static __le64 *irdma_puda_get_next_send_wqe(struct irdma_qp_uk *qp,
192 					    u32 *wqe_idx)
193 {
194 	int ret_code = 0;
195 
196 	*wqe_idx = IRDMA_RING_CURRENT_HEAD(qp->sq_ring);
197 	if (!*wqe_idx)
198 		qp->swqe_polarity = !qp->swqe_polarity;
199 	IRDMA_RING_MOVE_HEAD(qp->sq_ring, ret_code);
200 	if (ret_code)
201 		return NULL;
202 
203 	return qp->sq_base[*wqe_idx].elem;
204 }
205 
206 /**
207  * irdma_puda_poll_info - poll cq for completion
208  * @cq: cq for poll
209  * @info: info return for successful completion
210  */
irdma_puda_poll_info(struct irdma_sc_cq * cq,struct irdma_puda_cmpl_info * info)211 static int irdma_puda_poll_info(struct irdma_sc_cq *cq,
212 				struct irdma_puda_cmpl_info *info)
213 {
214 	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
215 	u64 qword0, qword2, qword3, qword6;
216 	__le64 *cqe;
217 	__le64 *ext_cqe = NULL;
218 	u64 qword7 = 0;
219 	u64 comp_ctx;
220 	bool valid_bit;
221 	bool ext_valid = 0;
222 	u32 major_err, minor_err;
223 	u32 peek_head;
224 	bool error;
225 	u8 polarity;
226 
227 	cqe = IRDMA_GET_CURRENT_CQ_ELEM(&cq->cq_uk);
228 	get_64bit_val(cqe, 24, &qword3);
229 	valid_bit = (bool)FIELD_GET(IRDMA_CQ_VALID, qword3);
230 	if (valid_bit != cq_uk->polarity)
231 		return -ENOENT;
232 
233 	if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
234 		ext_valid = (bool)FIELD_GET(IRDMA_CQ_EXTCQE, qword3);
235 
236 	if (ext_valid) {
237 		peek_head = (cq_uk->cq_ring.head + 1) % cq_uk->cq_ring.size;
238 		ext_cqe = cq_uk->cq_base[peek_head].buf;
239 		get_64bit_val(ext_cqe, 24, &qword7);
240 		polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword7);
241 		if (!peek_head)
242 			polarity ^= 1;
243 		if (polarity != cq_uk->polarity)
244 			return -ENOENT;
245 
246 		IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
247 		if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
248 			cq_uk->polarity = !cq_uk->polarity;
249 		/* update cq tail in cq shadow memory also */
250 		IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
251 	}
252 
253 	print_hex_dump_debug("PUDA: PUDA CQE", DUMP_PREFIX_OFFSET, 16, 8, cqe,
254 			     32, false);
255 	if (ext_valid)
256 		print_hex_dump_debug("PUDA: PUDA EXT-CQE", DUMP_PREFIX_OFFSET,
257 				     16, 8, ext_cqe, 32, false);
258 
259 	error = (bool)FIELD_GET(IRDMA_CQ_ERROR, qword3);
260 	if (error) {
261 		ibdev_dbg(to_ibdev(cq->dev), "PUDA: receive error\n");
262 		major_err = (u32)(FIELD_GET(IRDMA_CQ_MAJERR, qword3));
263 		minor_err = (u32)(FIELD_GET(IRDMA_CQ_MINERR, qword3));
264 		info->compl_error = major_err << 16 | minor_err;
265 		return -EIO;
266 	}
267 
268 	get_64bit_val(cqe, 0, &qword0);
269 	get_64bit_val(cqe, 16, &qword2);
270 
271 	info->q_type = (u8)FIELD_GET(IRDMA_CQ_SQ, qword3);
272 	info->qp_id = (u32)FIELD_GET(IRDMACQ_QPID, qword2);
273 	if (cq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
274 		info->ipv4 = (bool)FIELD_GET(IRDMACQ_IPV4, qword3);
275 
276 	get_64bit_val(cqe, 8, &comp_ctx);
277 	info->qp = (struct irdma_qp_uk *)(unsigned long)comp_ctx;
278 	info->wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, qword3);
279 
280 	if (info->q_type == IRDMA_CQE_QTYPE_RQ) {
281 		if (ext_valid) {
282 			info->vlan_valid = (bool)FIELD_GET(IRDMA_CQ_UDVLANVALID, qword7);
283 			if (info->vlan_valid) {
284 				get_64bit_val(ext_cqe, 16, &qword6);
285 				info->vlan = (u16)FIELD_GET(IRDMA_CQ_UDVLAN, qword6);
286 			}
287 			info->smac_valid = (bool)FIELD_GET(IRDMA_CQ_UDSMACVALID, qword7);
288 			if (info->smac_valid) {
289 				get_64bit_val(ext_cqe, 16, &qword6);
290 				info->smac[0] = (u8)((qword6 >> 40) & 0xFF);
291 				info->smac[1] = (u8)((qword6 >> 32) & 0xFF);
292 				info->smac[2] = (u8)((qword6 >> 24) & 0xFF);
293 				info->smac[3] = (u8)((qword6 >> 16) & 0xFF);
294 				info->smac[4] = (u8)((qword6 >> 8) & 0xFF);
295 				info->smac[5] = (u8)(qword6 & 0xFF);
296 			}
297 		}
298 
299 		if (cq->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
300 			info->vlan_valid = (bool)FIELD_GET(IRDMA_VLAN_TAG_VALID, qword3);
301 			info->l4proto = (u8)FIELD_GET(IRDMA_UDA_L4PROTO, qword2);
302 			info->l3proto = (u8)FIELD_GET(IRDMA_UDA_L3PROTO, qword2);
303 		}
304 
305 		info->payload_len = (u32)FIELD_GET(IRDMACQ_PAYLDLEN, qword0);
306 	}
307 
308 	return 0;
309 }
310 
311 /**
312  * irdma_puda_poll_cmpl - processes completion for cq
313  * @dev: iwarp device
314  * @cq: cq getting interrupt
315  * @compl_err: return any completion err
316  */
irdma_puda_poll_cmpl(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq,u32 * compl_err)317 int irdma_puda_poll_cmpl(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq,
318 			 u32 *compl_err)
319 {
320 	struct irdma_qp_uk *qp;
321 	struct irdma_cq_uk *cq_uk = &cq->cq_uk;
322 	struct irdma_puda_cmpl_info info = {};
323 	int ret = 0;
324 	struct irdma_puda_buf *buf;
325 	struct irdma_puda_rsrc *rsrc;
326 	u8 cq_type = cq->cq_type;
327 	unsigned long flags;
328 
329 	if (cq_type == IRDMA_CQ_TYPE_ILQ || cq_type == IRDMA_CQ_TYPE_IEQ) {
330 		rsrc = (cq_type == IRDMA_CQ_TYPE_ILQ) ? cq->vsi->ilq :
331 							cq->vsi->ieq;
332 	} else {
333 		ibdev_dbg(to_ibdev(dev), "PUDA: qp_type error\n");
334 		return -EINVAL;
335 	}
336 
337 	ret = irdma_puda_poll_info(cq, &info);
338 	*compl_err = info.compl_error;
339 	if (ret == -ENOENT)
340 		return ret;
341 	if (ret)
342 		goto done;
343 
344 	qp = info.qp;
345 	if (!qp || !rsrc) {
346 		ret = -EFAULT;
347 		goto done;
348 	}
349 
350 	if (qp->qp_id != rsrc->qp_id) {
351 		ret = -EFAULT;
352 		goto done;
353 	}
354 
355 	if (info.q_type == IRDMA_CQE_QTYPE_RQ) {
356 		buf = (struct irdma_puda_buf *)(uintptr_t)
357 			      qp->rq_wrid_array[info.wqe_idx];
358 
359 		/* reusing so synch the buffer for CPU use */
360 		dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
361 					buf->mem.size, DMA_BIDIRECTIONAL);
362 		/* Get all the tcpip information in the buf header */
363 		ret = irdma_puda_get_tcpip_info(&info, buf);
364 		if (ret) {
365 			rsrc->stats_rcvd_pkt_err++;
366 			if (cq_type == IRDMA_CQ_TYPE_ILQ) {
367 				irdma_ilq_putback_rcvbuf(&rsrc->qp, buf,
368 							 info.wqe_idx);
369 			} else {
370 				irdma_puda_ret_bufpool(rsrc, buf);
371 				irdma_puda_replenish_rq(rsrc, false);
372 			}
373 			goto done;
374 		}
375 
376 		rsrc->stats_pkt_rcvd++;
377 		rsrc->compl_rxwqe_idx = info.wqe_idx;
378 		ibdev_dbg(to_ibdev(dev), "PUDA: RQ completion\n");
379 		rsrc->receive(rsrc->vsi, buf);
380 		if (cq_type == IRDMA_CQ_TYPE_ILQ)
381 			irdma_ilq_putback_rcvbuf(&rsrc->qp, buf, info.wqe_idx);
382 		else
383 			irdma_puda_replenish_rq(rsrc, false);
384 
385 	} else {
386 		ibdev_dbg(to_ibdev(dev), "PUDA: SQ completion\n");
387 		buf = (struct irdma_puda_buf *)(uintptr_t)
388 					qp->sq_wrtrk_array[info.wqe_idx].wrid;
389 
390 		/* reusing so synch the buffer for CPU use */
391 		dma_sync_single_for_cpu(dev->hw->device, buf->mem.pa,
392 					buf->mem.size, DMA_BIDIRECTIONAL);
393 		IRDMA_RING_SET_TAIL(qp->sq_ring, info.wqe_idx);
394 		rsrc->xmit_complete(rsrc->vsi, buf);
395 		spin_lock_irqsave(&rsrc->bufpool_lock, flags);
396 		rsrc->tx_wqe_avail_cnt++;
397 		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
398 		if (!list_empty(&rsrc->txpend))
399 			irdma_puda_send_buf(rsrc, NULL);
400 	}
401 
402 done:
403 	IRDMA_RING_MOVE_HEAD_NOCHECK(cq_uk->cq_ring);
404 	if (!IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring))
405 		cq_uk->polarity = !cq_uk->polarity;
406 	/* update cq tail in cq shadow memory also */
407 	IRDMA_RING_MOVE_TAIL(cq_uk->cq_ring);
408 	set_64bit_val(cq_uk->shadow_area, 0,
409 		      IRDMA_RING_CURRENT_HEAD(cq_uk->cq_ring));
410 
411 	return ret;
412 }
413 
414 /**
415  * irdma_puda_send - complete send wqe for transmit
416  * @qp: puda qp for send
417  * @info: buffer information for transmit
418  */
irdma_puda_send(struct irdma_sc_qp * qp,struct irdma_puda_send_info * info)419 int irdma_puda_send(struct irdma_sc_qp *qp, struct irdma_puda_send_info *info)
420 {
421 	__le64 *wqe;
422 	u32 iplen, l4len;
423 	u64 hdr[2];
424 	u32 wqe_idx;
425 	u8 iipt;
426 
427 	/* number of 32 bits DWORDS in header */
428 	l4len = info->tcplen >> 2;
429 	if (info->ipv4) {
430 		iipt = 3;
431 		iplen = 5;
432 	} else {
433 		iipt = 1;
434 		iplen = 10;
435 	}
436 
437 	wqe = irdma_puda_get_next_send_wqe(&qp->qp_uk, &wqe_idx);
438 	if (!wqe)
439 		return -ENOMEM;
440 
441 	qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid = (uintptr_t)info->scratch;
442 	/* Third line of WQE descriptor */
443 	/* maclen is in words */
444 
445 	if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
446 		hdr[0] = 0; /* Dest_QPN and Dest_QKey only for UD */
447 		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
448 			 FIELD_PREP(IRDMA_UDA_QPSQ_L4LEN, l4len) |
449 			 FIELD_PREP(IRDMAQPSQ_AHID, info->ah_id) |
450 			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
451 			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID,
452 				    qp->qp_uk.swqe_polarity);
453 
454 		/* Forth line of WQE descriptor */
455 
456 		set_64bit_val(wqe, 0, info->paddr);
457 		set_64bit_val(wqe, 8,
458 			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, info->len) |
459 			      FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity));
460 	} else {
461 		hdr[0] = FIELD_PREP(IRDMA_UDA_QPSQ_MACLEN, info->maclen >> 1) |
462 			 FIELD_PREP(IRDMA_UDA_QPSQ_IPLEN, iplen) |
463 			 FIELD_PREP(IRDMA_UDA_QPSQ_L4T, 1) |
464 			 FIELD_PREP(IRDMA_UDA_QPSQ_IIPT, iipt) |
465 			 FIELD_PREP(IRDMA_GEN1_UDA_QPSQ_L4LEN, l4len);
466 
467 		hdr[1] = FIELD_PREP(IRDMA_UDA_QPSQ_OPCODE, IRDMA_OP_TYPE_SEND) |
468 			 FIELD_PREP(IRDMA_UDA_QPSQ_SIGCOMPL, 1) |
469 			 FIELD_PREP(IRDMA_UDA_QPSQ_DOLOOPBACK, info->do_lpb) |
470 			 FIELD_PREP(IRDMA_UDA_QPSQ_VALID, qp->qp_uk.swqe_polarity);
471 
472 		/* Forth line of WQE descriptor */
473 
474 		set_64bit_val(wqe, 0, info->paddr);
475 		set_64bit_val(wqe, 8,
476 			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, info->len));
477 	}
478 
479 	set_64bit_val(wqe, 16, hdr[0]);
480 	dma_wmb(); /* make sure WQE is written before valid bit is set */
481 
482 	set_64bit_val(wqe, 24, hdr[1]);
483 
484 	print_hex_dump_debug("PUDA: PUDA SEND WQE", DUMP_PREFIX_OFFSET, 16, 8,
485 			     wqe, 32, false);
486 	irdma_uk_qp_post_wr(&qp->qp_uk);
487 	return 0;
488 }
489 
490 /**
491  * irdma_puda_send_buf - transmit puda buffer
492  * @rsrc: resource to use for buffer
493  * @buf: puda buffer to transmit
494  */
irdma_puda_send_buf(struct irdma_puda_rsrc * rsrc,struct irdma_puda_buf * buf)495 void irdma_puda_send_buf(struct irdma_puda_rsrc *rsrc,
496 			 struct irdma_puda_buf *buf)
497 {
498 	struct irdma_puda_send_info info;
499 	int ret = 0;
500 	unsigned long flags;
501 
502 	spin_lock_irqsave(&rsrc->bufpool_lock, flags);
503 	/* if no wqe available or not from a completion and we have
504 	 * pending buffers, we must queue new buffer
505 	 */
506 	if (!rsrc->tx_wqe_avail_cnt || (buf && !list_empty(&rsrc->txpend))) {
507 		list_add_tail(&buf->list, &rsrc->txpend);
508 		spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
509 		rsrc->stats_sent_pkt_q++;
510 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
511 			ibdev_dbg(to_ibdev(rsrc->dev),
512 				  "PUDA: adding to txpend\n");
513 		return;
514 	}
515 	rsrc->tx_wqe_avail_cnt--;
516 	/* if we are coming from a completion and have pending buffers
517 	 * then Get one from pending list
518 	 */
519 	if (!buf) {
520 		buf = irdma_puda_get_listbuf(&rsrc->txpend);
521 		if (!buf)
522 			goto done;
523 	}
524 
525 	info.scratch = buf;
526 	info.paddr = buf->mem.pa;
527 	info.len = buf->totallen;
528 	info.tcplen = buf->tcphlen;
529 	info.ipv4 = buf->ipv4;
530 
531 	if (rsrc->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
532 		info.ah_id = buf->ah_id;
533 	} else {
534 		info.maclen = buf->maclen;
535 		info.do_lpb = buf->do_lpb;
536 	}
537 
538 	/* Synch buffer for use by device */
539 	dma_sync_single_for_cpu(rsrc->dev->hw->device, buf->mem.pa,
540 				buf->mem.size, DMA_BIDIRECTIONAL);
541 	ret = irdma_puda_send(&rsrc->qp, &info);
542 	if (ret) {
543 		rsrc->tx_wqe_avail_cnt++;
544 		rsrc->stats_sent_pkt_q++;
545 		list_add(&buf->list, &rsrc->txpend);
546 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
547 			ibdev_dbg(to_ibdev(rsrc->dev),
548 				  "PUDA: adding to puda_send\n");
549 	} else {
550 		rsrc->stats_pkt_sent++;
551 	}
552 done:
553 	spin_unlock_irqrestore(&rsrc->bufpool_lock, flags);
554 }
555 
556 /**
557  * irdma_puda_qp_setctx - during init, set qp's context
558  * @rsrc: qp's resource
559  */
irdma_puda_qp_setctx(struct irdma_puda_rsrc * rsrc)560 static void irdma_puda_qp_setctx(struct irdma_puda_rsrc *rsrc)
561 {
562 	struct irdma_sc_qp *qp = &rsrc->qp;
563 	__le64 *qp_ctx = qp->hw_host_ctx;
564 
565 	set_64bit_val(qp_ctx, 8, qp->sq_pa);
566 	set_64bit_val(qp_ctx, 16, qp->rq_pa);
567 	set_64bit_val(qp_ctx, 24,
568 		      FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
569 		      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size));
570 	set_64bit_val(qp_ctx, 48,
571 		      FIELD_PREP(IRDMAQPC_SNDMSS, rsrc->buf_size));
572 	set_64bit_val(qp_ctx, 56, 0);
573 	if (qp->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
574 		set_64bit_val(qp_ctx, 64, 1);
575 	set_64bit_val(qp_ctx, 136,
576 		      FIELD_PREP(IRDMAQPC_TXCQNUM, rsrc->cq_id) |
577 		      FIELD_PREP(IRDMAQPC_RXCQNUM, rsrc->cq_id));
578 	set_64bit_val(qp_ctx, 144,
579 		      FIELD_PREP(IRDMAQPC_STAT_INDEX, rsrc->stats_idx));
580 	set_64bit_val(qp_ctx, 160,
581 		      FIELD_PREP(IRDMAQPC_PRIVEN, 1) |
582 		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, rsrc->stats_idx_valid));
583 	set_64bit_val(qp_ctx, 168,
584 		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, (uintptr_t)qp));
585 	set_64bit_val(qp_ctx, 176,
586 		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
587 		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
588 		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
589 
590 	print_hex_dump_debug("PUDA: PUDA QP CONTEXT", DUMP_PREFIX_OFFSET, 16,
591 			     8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
592 }
593 
594 /**
595  * irdma_puda_qp_wqe - setup wqe for qp create
596  * @dev: Device
597  * @qp: Resource qp
598  */
irdma_puda_qp_wqe(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)599 static int irdma_puda_qp_wqe(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
600 {
601 	struct irdma_sc_cqp *cqp;
602 	__le64 *wqe;
603 	u64 hdr;
604 	struct irdma_ccq_cqe_info compl_info;
605 	int status = 0;
606 
607 	cqp = dev->cqp;
608 	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
609 	if (!wqe)
610 		return -ENOMEM;
611 
612 	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
613 	set_64bit_val(wqe, 40, qp->shadow_area_pa);
614 
615 	hdr = qp->qp_uk.qp_id |
616 	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
617 	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, IRDMA_QP_TYPE_UDA) |
618 	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, 1) |
619 	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, 2) |
620 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
621 	dma_wmb(); /* make sure WQE is written before valid bit is set */
622 
623 	set_64bit_val(wqe, 24, hdr);
624 
625 	print_hex_dump_debug("PUDA: PUDA QP CREATE", DUMP_PREFIX_OFFSET, 16,
626 			     8, wqe, 40, false);
627 	irdma_sc_cqp_post_sq(cqp);
628 	status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_QP,
629 					       &compl_info);
630 
631 	return status;
632 }
633 
634 /**
635  * irdma_puda_qp_create - create qp for resource
636  * @rsrc: resource to use for buffer
637  */
irdma_puda_qp_create(struct irdma_puda_rsrc * rsrc)638 static int irdma_puda_qp_create(struct irdma_puda_rsrc *rsrc)
639 {
640 	struct irdma_sc_qp *qp = &rsrc->qp;
641 	struct irdma_qp_uk *ukqp = &qp->qp_uk;
642 	int ret = 0;
643 	u32 sq_size, rq_size;
644 	struct irdma_dma_mem *mem;
645 
646 	sq_size = rsrc->sq_size * IRDMA_QP_WQE_MIN_SIZE;
647 	rq_size = rsrc->rq_size * IRDMA_QP_WQE_MIN_SIZE;
648 	rsrc->qpmem.size = ALIGN((sq_size + rq_size + (IRDMA_SHADOW_AREA_SIZE << 3) + IRDMA_QP_CTX_SIZE),
649 				 IRDMA_HW_PAGE_SIZE);
650 	rsrc->qpmem.va = dma_alloc_coherent(rsrc->dev->hw->device,
651 					    rsrc->qpmem.size, &rsrc->qpmem.pa,
652 					    GFP_KERNEL);
653 	if (!rsrc->qpmem.va)
654 		return -ENOMEM;
655 
656 	mem = &rsrc->qpmem;
657 	memset(mem->va, 0, rsrc->qpmem.size);
658 	qp->hw_sq_size = irdma_get_encoded_wqe_size(rsrc->sq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
659 	qp->hw_rq_size = irdma_get_encoded_wqe_size(rsrc->rq_size, IRDMA_QUEUE_TYPE_SQ_RQ);
660 	qp->pd = &rsrc->sc_pd;
661 	qp->qp_uk.qp_type = IRDMA_QP_TYPE_UDA;
662 	qp->dev = rsrc->dev;
663 	qp->qp_uk.back_qp = rsrc;
664 	qp->sq_pa = mem->pa;
665 	qp->rq_pa = qp->sq_pa + sq_size;
666 	qp->vsi = rsrc->vsi;
667 	ukqp->sq_base = mem->va;
668 	ukqp->rq_base = &ukqp->sq_base[rsrc->sq_size];
669 	ukqp->shadow_area = ukqp->rq_base[rsrc->rq_size].elem;
670 	ukqp->uk_attrs = &qp->dev->hw_attrs.uk_attrs;
671 	qp->shadow_area_pa = qp->rq_pa + rq_size;
672 	qp->hw_host_ctx = ukqp->shadow_area + IRDMA_SHADOW_AREA_SIZE;
673 	qp->hw_host_ctx_pa = qp->shadow_area_pa + (IRDMA_SHADOW_AREA_SIZE << 3);
674 	qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
675 	ukqp->qp_id = rsrc->qp_id;
676 	ukqp->sq_wrtrk_array = rsrc->sq_wrtrk_array;
677 	ukqp->rq_wrid_array = rsrc->rq_wrid_array;
678 	ukqp->sq_size = rsrc->sq_size;
679 	ukqp->rq_size = rsrc->rq_size;
680 
681 	IRDMA_RING_INIT(ukqp->sq_ring, ukqp->sq_size);
682 	IRDMA_RING_INIT(ukqp->initial_ring, ukqp->sq_size);
683 	IRDMA_RING_INIT(ukqp->rq_ring, ukqp->rq_size);
684 	ukqp->wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
685 
686 	ret = rsrc->dev->ws_add(qp->vsi, qp->user_pri);
687 	if (ret) {
688 		dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
689 				  rsrc->qpmem.va, rsrc->qpmem.pa);
690 		rsrc->qpmem.va = NULL;
691 		return ret;
692 	}
693 
694 	irdma_qp_add_qos(qp);
695 	irdma_puda_qp_setctx(rsrc);
696 
697 	if (rsrc->dev->ceq_valid)
698 		ret = irdma_cqp_qp_create_cmd(rsrc->dev, qp);
699 	else
700 		ret = irdma_puda_qp_wqe(rsrc->dev, qp);
701 	if (ret) {
702 		irdma_qp_rem_qos(qp);
703 		rsrc->dev->ws_remove(qp->vsi, qp->user_pri);
704 		dma_free_coherent(rsrc->dev->hw->device, rsrc->qpmem.size,
705 				  rsrc->qpmem.va, rsrc->qpmem.pa);
706 		rsrc->qpmem.va = NULL;
707 	}
708 
709 	return ret;
710 }
711 
712 /**
713  * irdma_puda_cq_wqe - setup wqe for CQ create
714  * @dev: Device
715  * @cq: resource for cq
716  */
irdma_puda_cq_wqe(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)717 static int irdma_puda_cq_wqe(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
718 {
719 	__le64 *wqe;
720 	struct irdma_sc_cqp *cqp;
721 	u64 hdr;
722 	struct irdma_ccq_cqe_info compl_info;
723 	int status = 0;
724 
725 	cqp = dev->cqp;
726 	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, 0);
727 	if (!wqe)
728 		return -ENOMEM;
729 
730 	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
731 	set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
732 	set_64bit_val(wqe, 16,
733 		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
734 	set_64bit_val(wqe, 32, cq->cq_pa);
735 	set_64bit_val(wqe, 40, cq->shadow_area_pa);
736 	set_64bit_val(wqe, 56,
737 		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
738 		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
739 
740 	hdr = cq->cq_uk.cq_id |
741 	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
742 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, 1) |
743 	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, 1) |
744 	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, 1) |
745 	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
746 	dma_wmb(); /* make sure WQE is written before valid bit is set */
747 
748 	set_64bit_val(wqe, 24, hdr);
749 
750 	print_hex_dump_debug("PUDA: PUDA CREATE CQ", DUMP_PREFIX_OFFSET, 16,
751 			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
752 	irdma_sc_cqp_post_sq(dev->cqp);
753 	status = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_CREATE_CQ,
754 					       &compl_info);
755 	if (!status) {
756 		struct irdma_sc_ceq *ceq = dev->ceq[0];
757 
758 		if (ceq && ceq->reg_cq)
759 			status = irdma_sc_add_cq_ctx(ceq, cq);
760 	}
761 
762 	return status;
763 }
764 
765 /**
766  * irdma_puda_cq_create - create cq for resource
767  * @rsrc: resource for which cq to create
768  */
irdma_puda_cq_create(struct irdma_puda_rsrc * rsrc)769 static int irdma_puda_cq_create(struct irdma_puda_rsrc *rsrc)
770 {
771 	struct irdma_sc_dev *dev = rsrc->dev;
772 	struct irdma_sc_cq *cq = &rsrc->cq;
773 	int ret = 0;
774 	u32 cqsize;
775 	struct irdma_dma_mem *mem;
776 	struct irdma_cq_init_info info = {};
777 	struct irdma_cq_uk_init_info *init_info = &info.cq_uk_init_info;
778 
779 	cq->vsi = rsrc->vsi;
780 	cqsize = rsrc->cq_size * (sizeof(struct irdma_cqe));
781 	rsrc->cqmem.size = ALIGN(cqsize + sizeof(struct irdma_cq_shadow_area),
782 				 IRDMA_CQ0_ALIGNMENT);
783 	rsrc->cqmem.va = dma_alloc_coherent(dev->hw->device, rsrc->cqmem.size,
784 					    &rsrc->cqmem.pa, GFP_KERNEL);
785 	if (!rsrc->cqmem.va)
786 		return -ENOMEM;
787 
788 	mem = &rsrc->cqmem;
789 	info.dev = dev;
790 	info.type = (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ) ?
791 		    IRDMA_CQ_TYPE_ILQ : IRDMA_CQ_TYPE_IEQ;
792 	info.shadow_read_threshold = rsrc->cq_size >> 2;
793 	info.cq_base_pa = mem->pa;
794 	info.shadow_area_pa = mem->pa + cqsize;
795 	init_info->cq_base = mem->va;
796 	init_info->shadow_area = (__le64 *)((u8 *)mem->va + cqsize);
797 	init_info->cq_size = rsrc->cq_size;
798 	init_info->cq_id = rsrc->cq_id;
799 	info.ceqe_mask = true;
800 	info.ceq_id_valid = true;
801 	info.vsi = rsrc->vsi;
802 
803 	ret = irdma_sc_cq_init(cq, &info);
804 	if (ret)
805 		goto error;
806 
807 	if (rsrc->dev->ceq_valid)
808 		ret = irdma_cqp_cq_create_cmd(dev, cq);
809 	else
810 		ret = irdma_puda_cq_wqe(dev, cq);
811 error:
812 	if (ret) {
813 		dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
814 				  rsrc->cqmem.va, rsrc->cqmem.pa);
815 		rsrc->cqmem.va = NULL;
816 	}
817 
818 	return ret;
819 }
820 
821 /**
822  * irdma_puda_free_qp - free qp for resource
823  * @rsrc: resource for which qp to free
824  */
irdma_puda_free_qp(struct irdma_puda_rsrc * rsrc)825 static void irdma_puda_free_qp(struct irdma_puda_rsrc *rsrc)
826 {
827 	int ret;
828 	struct irdma_ccq_cqe_info compl_info;
829 	struct irdma_sc_dev *dev = rsrc->dev;
830 
831 	if (rsrc->dev->ceq_valid) {
832 		irdma_cqp_qp_destroy_cmd(dev, &rsrc->qp);
833 		rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
834 		return;
835 	}
836 
837 	ret = irdma_sc_qp_destroy(&rsrc->qp, 0, false, true, true);
838 	if (ret)
839 		ibdev_dbg(to_ibdev(dev),
840 			  "PUDA: error puda qp destroy wqe, status = %d\n",
841 			  ret);
842 	if (!ret) {
843 		ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_QP,
844 						    &compl_info);
845 		if (ret)
846 			ibdev_dbg(to_ibdev(dev),
847 				  "PUDA: error puda qp destroy failed, status = %d\n",
848 				  ret);
849 	}
850 	rsrc->dev->ws_remove(rsrc->qp.vsi, rsrc->qp.user_pri);
851 }
852 
853 /**
854  * irdma_puda_free_cq - free cq for resource
855  * @rsrc: resource for which cq to free
856  */
irdma_puda_free_cq(struct irdma_puda_rsrc * rsrc)857 static void irdma_puda_free_cq(struct irdma_puda_rsrc *rsrc)
858 {
859 	int ret;
860 	struct irdma_ccq_cqe_info compl_info;
861 	struct irdma_sc_dev *dev = rsrc->dev;
862 
863 	if (rsrc->dev->ceq_valid) {
864 		irdma_cqp_cq_destroy_cmd(dev, &rsrc->cq);
865 		return;
866 	}
867 
868 	ret = irdma_sc_cq_destroy(&rsrc->cq, 0, true);
869 	if (ret)
870 		ibdev_dbg(to_ibdev(dev), "PUDA: error ieq cq destroy\n");
871 	if (!ret) {
872 		ret = irdma_sc_poll_for_cqp_op_done(dev->cqp, IRDMA_CQP_OP_DESTROY_CQ,
873 						    &compl_info);
874 		if (ret)
875 			ibdev_dbg(to_ibdev(dev),
876 				  "PUDA: error ieq qp destroy done\n");
877 	}
878 }
879 
880 /**
881  * irdma_puda_dele_rsrc - delete all resources during close
882  * @vsi: VSI structure of device
883  * @type: type of resource to dele
884  * @reset: true if reset chip
885  */
irdma_puda_dele_rsrc(struct irdma_sc_vsi * vsi,enum puda_rsrc_type type,bool reset)886 void irdma_puda_dele_rsrc(struct irdma_sc_vsi *vsi, enum puda_rsrc_type type,
887 			  bool reset)
888 {
889 	struct irdma_sc_dev *dev = vsi->dev;
890 	struct irdma_puda_rsrc *rsrc;
891 	struct irdma_puda_buf *buf = NULL;
892 	struct irdma_puda_buf *nextbuf = NULL;
893 	struct irdma_virt_mem *vmem;
894 	struct irdma_sc_ceq *ceq;
895 
896 	ceq = vsi->dev->ceq[0];
897 	switch (type) {
898 	case IRDMA_PUDA_RSRC_TYPE_ILQ:
899 		rsrc = vsi->ilq;
900 		vmem = &vsi->ilq_mem;
901 		vsi->ilq = NULL;
902 		if (ceq && ceq->reg_cq)
903 			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
904 		break;
905 	case IRDMA_PUDA_RSRC_TYPE_IEQ:
906 		rsrc = vsi->ieq;
907 		vmem = &vsi->ieq_mem;
908 		vsi->ieq = NULL;
909 		if (ceq && ceq->reg_cq)
910 			irdma_sc_remove_cq_ctx(ceq, &rsrc->cq);
911 		break;
912 	default:
913 		ibdev_dbg(to_ibdev(dev), "PUDA: error resource type = 0x%x\n",
914 			  type);
915 		return;
916 	}
917 
918 	switch (rsrc->cmpl) {
919 	case PUDA_HASH_CRC_COMPLETE:
920 		irdma_free_hash_desc(rsrc->hash_desc);
921 		fallthrough;
922 	case PUDA_QP_CREATED:
923 		irdma_qp_rem_qos(&rsrc->qp);
924 
925 		if (!reset)
926 			irdma_puda_free_qp(rsrc);
927 
928 		dma_free_coherent(dev->hw->device, rsrc->qpmem.size,
929 				  rsrc->qpmem.va, rsrc->qpmem.pa);
930 		rsrc->qpmem.va = NULL;
931 		fallthrough;
932 	case PUDA_CQ_CREATED:
933 		if (!reset)
934 			irdma_puda_free_cq(rsrc);
935 
936 		dma_free_coherent(dev->hw->device, rsrc->cqmem.size,
937 				  rsrc->cqmem.va, rsrc->cqmem.pa);
938 		rsrc->cqmem.va = NULL;
939 		break;
940 	default:
941 		ibdev_dbg(to_ibdev(rsrc->dev), "PUDA: error no resources\n");
942 		break;
943 	}
944 	/* Free all allocated puda buffers for both tx and rx */
945 	buf = rsrc->alloclist;
946 	while (buf) {
947 		nextbuf = buf->next;
948 		irdma_puda_dele_buf(dev, buf);
949 		buf = nextbuf;
950 		rsrc->alloc_buf_count--;
951 	}
952 
953 	kfree(vmem->va);
954 }
955 
956 /**
957  * irdma_puda_allocbufs - allocate buffers for resource
958  * @rsrc: resource for buffer allocation
959  * @count: number of buffers to create
960  */
irdma_puda_allocbufs(struct irdma_puda_rsrc * rsrc,u32 count)961 static int irdma_puda_allocbufs(struct irdma_puda_rsrc *rsrc, u32 count)
962 {
963 	u32 i;
964 	struct irdma_puda_buf *buf;
965 	struct irdma_puda_buf *nextbuf;
966 
967 	for (i = 0; i < count; i++) {
968 		buf = irdma_puda_alloc_buf(rsrc->dev, rsrc->buf_size);
969 		if (!buf) {
970 			rsrc->stats_buf_alloc_fail++;
971 			return -ENOMEM;
972 		}
973 		irdma_puda_ret_bufpool(rsrc, buf);
974 		rsrc->alloc_buf_count++;
975 		if (!rsrc->alloclist) {
976 			rsrc->alloclist = buf;
977 		} else {
978 			nextbuf = rsrc->alloclist;
979 			rsrc->alloclist = buf;
980 			buf->next = nextbuf;
981 		}
982 	}
983 
984 	rsrc->avail_buf_count = rsrc->alloc_buf_count;
985 
986 	return 0;
987 }
988 
989 /**
990  * irdma_puda_create_rsrc - create resource (ilq or ieq)
991  * @vsi: sc VSI struct
992  * @info: resource information
993  */
irdma_puda_create_rsrc(struct irdma_sc_vsi * vsi,struct irdma_puda_rsrc_info * info)994 int irdma_puda_create_rsrc(struct irdma_sc_vsi *vsi,
995 			   struct irdma_puda_rsrc_info *info)
996 {
997 	struct irdma_sc_dev *dev = vsi->dev;
998 	int ret = 0;
999 	struct irdma_puda_rsrc *rsrc;
1000 	u32 pudasize;
1001 	u32 sqwridsize, rqwridsize;
1002 	struct irdma_virt_mem *vmem;
1003 
1004 	info->count = 1;
1005 	pudasize = sizeof(struct irdma_puda_rsrc);
1006 	sqwridsize = info->sq_size * sizeof(struct irdma_sq_uk_wr_trk_info);
1007 	rqwridsize = info->rq_size * 8;
1008 	switch (info->type) {
1009 	case IRDMA_PUDA_RSRC_TYPE_ILQ:
1010 		vmem = &vsi->ilq_mem;
1011 		break;
1012 	case IRDMA_PUDA_RSRC_TYPE_IEQ:
1013 		vmem = &vsi->ieq_mem;
1014 		break;
1015 	default:
1016 		return -EOPNOTSUPP;
1017 	}
1018 	vmem->size = pudasize + sqwridsize + rqwridsize;
1019 	vmem->va = kzalloc(vmem->size, GFP_KERNEL);
1020 	if (!vmem->va)
1021 		return -ENOMEM;
1022 
1023 	rsrc = vmem->va;
1024 	spin_lock_init(&rsrc->bufpool_lock);
1025 	switch (info->type) {
1026 	case IRDMA_PUDA_RSRC_TYPE_ILQ:
1027 		vsi->ilq = vmem->va;
1028 		vsi->ilq_count = info->count;
1029 		rsrc->receive = info->receive;
1030 		rsrc->xmit_complete = info->xmit_complete;
1031 		break;
1032 	case IRDMA_PUDA_RSRC_TYPE_IEQ:
1033 		vsi->ieq_count = info->count;
1034 		vsi->ieq = vmem->va;
1035 		rsrc->receive = irdma_ieq_receive;
1036 		rsrc->xmit_complete = irdma_ieq_tx_compl;
1037 		break;
1038 	default:
1039 		return -EOPNOTSUPP;
1040 	}
1041 
1042 	rsrc->type = info->type;
1043 	rsrc->sq_wrtrk_array = (struct irdma_sq_uk_wr_trk_info *)
1044 			       ((u8 *)vmem->va + pudasize);
1045 	rsrc->rq_wrid_array = (u64 *)((u8 *)vmem->va + pudasize + sqwridsize);
1046 	/* Initialize all ieq lists */
1047 	INIT_LIST_HEAD(&rsrc->bufpool);
1048 	INIT_LIST_HEAD(&rsrc->txpend);
1049 
1050 	rsrc->tx_wqe_avail_cnt = info->sq_size - 1;
1051 	irdma_sc_pd_init(dev, &rsrc->sc_pd, info->pd_id, info->abi_ver);
1052 	rsrc->qp_id = info->qp_id;
1053 	rsrc->cq_id = info->cq_id;
1054 	rsrc->sq_size = info->sq_size;
1055 	rsrc->rq_size = info->rq_size;
1056 	rsrc->cq_size = info->rq_size + info->sq_size;
1057 	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1058 		if (rsrc->type == IRDMA_PUDA_RSRC_TYPE_ILQ)
1059 			rsrc->cq_size += info->rq_size;
1060 	}
1061 	rsrc->buf_size = info->buf_size;
1062 	rsrc->dev = dev;
1063 	rsrc->vsi = vsi;
1064 	rsrc->stats_idx = info->stats_idx;
1065 	rsrc->stats_idx_valid = info->stats_idx_valid;
1066 
1067 	ret = irdma_puda_cq_create(rsrc);
1068 	if (!ret) {
1069 		rsrc->cmpl = PUDA_CQ_CREATED;
1070 		ret = irdma_puda_qp_create(rsrc);
1071 	}
1072 	if (ret) {
1073 		ibdev_dbg(to_ibdev(dev),
1074 			  "PUDA: error qp_create type=%d, status=%d\n",
1075 			  rsrc->type, ret);
1076 		goto error;
1077 	}
1078 	rsrc->cmpl = PUDA_QP_CREATED;
1079 
1080 	ret = irdma_puda_allocbufs(rsrc, info->tx_buf_cnt + info->rq_size);
1081 	if (ret) {
1082 		ibdev_dbg(to_ibdev(dev), "PUDA: error alloc_buf\n");
1083 		goto error;
1084 	}
1085 
1086 	rsrc->rxq_invalid_cnt = info->rq_size;
1087 	ret = irdma_puda_replenish_rq(rsrc, true);
1088 	if (ret)
1089 		goto error;
1090 
1091 	if (info->type == IRDMA_PUDA_RSRC_TYPE_IEQ) {
1092 		if (!irdma_init_hash_desc(&rsrc->hash_desc)) {
1093 			rsrc->check_crc = true;
1094 			rsrc->cmpl = PUDA_HASH_CRC_COMPLETE;
1095 			ret = 0;
1096 		}
1097 	}
1098 
1099 	irdma_sc_ccq_arm(&rsrc->cq);
1100 	return ret;
1101 
1102 error:
1103 	irdma_puda_dele_rsrc(vsi, info->type, false);
1104 
1105 	return ret;
1106 }
1107 
1108 /**
1109  * irdma_ilq_putback_rcvbuf - ilq buffer to put back on rq
1110  * @qp: ilq's qp resource
1111  * @buf: puda buffer for rcv q
1112  * @wqe_idx:  wqe index of completed rcvbuf
1113  */
irdma_ilq_putback_rcvbuf(struct irdma_sc_qp * qp,struct irdma_puda_buf * buf,u32 wqe_idx)1114 static void irdma_ilq_putback_rcvbuf(struct irdma_sc_qp *qp,
1115 				     struct irdma_puda_buf *buf, u32 wqe_idx)
1116 {
1117 	__le64 *wqe;
1118 	u64 offset8, offset24;
1119 
1120 	/* Synch buffer for use by device */
1121 	dma_sync_single_for_device(qp->dev->hw->device, buf->mem.pa,
1122 				   buf->mem.size, DMA_BIDIRECTIONAL);
1123 	wqe = qp->qp_uk.rq_base[wqe_idx].elem;
1124 	get_64bit_val(wqe, 24, &offset24);
1125 	if (qp->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1126 		get_64bit_val(wqe, 8, &offset8);
1127 		if (offset24)
1128 			offset8 &= ~FIELD_PREP(IRDMAQPSQ_VALID, 1);
1129 		else
1130 			offset8 |= FIELD_PREP(IRDMAQPSQ_VALID, 1);
1131 		set_64bit_val(wqe, 8, offset8);
1132 		dma_wmb(); /* make sure WQE is written before valid bit is set */
1133 	}
1134 	if (offset24)
1135 		offset24 = 0;
1136 	else
1137 		offset24 = FIELD_PREP(IRDMAQPSQ_VALID, 1);
1138 
1139 	set_64bit_val(wqe, 24, offset24);
1140 }
1141 
1142 /**
1143  * irdma_ieq_get_fpdu_len - get length of fpdu with or without marker
1144  * @pfpdu: pointer to fpdu
1145  * @datap: pointer to data in the buffer
1146  * @rcv_seq: seqnum of the data buffer
1147  */
irdma_ieq_get_fpdu_len(struct irdma_pfpdu * pfpdu,u8 * datap,u32 rcv_seq)1148 static u16 irdma_ieq_get_fpdu_len(struct irdma_pfpdu *pfpdu, u8 *datap,
1149 				  u32 rcv_seq)
1150 {
1151 	u32 marker_seq, end_seq, blk_start;
1152 	u8 marker_len = pfpdu->marker_len;
1153 	u16 total_len = 0;
1154 	u16 fpdu_len;
1155 
1156 	blk_start = (pfpdu->rcv_start_seq - rcv_seq) & (IRDMA_MRK_BLK_SZ - 1);
1157 	if (!blk_start) {
1158 		total_len = marker_len;
1159 		marker_seq = rcv_seq + IRDMA_MRK_BLK_SZ;
1160 		if (marker_len && *(u32 *)datap)
1161 			return 0;
1162 	} else {
1163 		marker_seq = rcv_seq + blk_start;
1164 	}
1165 
1166 	datap += total_len;
1167 	fpdu_len = ntohs(*(__be16 *)datap);
1168 	fpdu_len += IRDMA_IEQ_MPA_FRAMING;
1169 	fpdu_len = (fpdu_len + 3) & 0xfffc;
1170 
1171 	if (fpdu_len > pfpdu->max_fpdu_data)
1172 		return 0;
1173 
1174 	total_len += fpdu_len;
1175 	end_seq = rcv_seq + total_len;
1176 	while ((int)(marker_seq - end_seq) < 0) {
1177 		total_len += marker_len;
1178 		end_seq += marker_len;
1179 		marker_seq += IRDMA_MRK_BLK_SZ;
1180 	}
1181 
1182 	return total_len;
1183 }
1184 
1185 /**
1186  * irdma_ieq_copy_to_txbuf - copydata from rcv buf to tx buf
1187  * @buf: rcv buffer with partial
1188  * @txbuf: tx buffer for sending back
1189  * @buf_offset: rcv buffer offset to copy from
1190  * @txbuf_offset: at offset in tx buf to copy
1191  * @len: length of data to copy
1192  */
irdma_ieq_copy_to_txbuf(struct irdma_puda_buf * buf,struct irdma_puda_buf * txbuf,u16 buf_offset,u32 txbuf_offset,u32 len)1193 static void irdma_ieq_copy_to_txbuf(struct irdma_puda_buf *buf,
1194 				    struct irdma_puda_buf *txbuf,
1195 				    u16 buf_offset, u32 txbuf_offset, u32 len)
1196 {
1197 	void *mem1 = (u8 *)buf->mem.va + buf_offset;
1198 	void *mem2 = (u8 *)txbuf->mem.va + txbuf_offset;
1199 
1200 	memcpy(mem2, mem1, len);
1201 }
1202 
1203 /**
1204  * irdma_ieq_setup_tx_buf - setup tx buffer for partial handling
1205  * @buf: reeive buffer with partial
1206  * @txbuf: buffer to prepare
1207  */
irdma_ieq_setup_tx_buf(struct irdma_puda_buf * buf,struct irdma_puda_buf * txbuf)1208 static void irdma_ieq_setup_tx_buf(struct irdma_puda_buf *buf,
1209 				   struct irdma_puda_buf *txbuf)
1210 {
1211 	txbuf->tcphlen = buf->tcphlen;
1212 	txbuf->ipv4 = buf->ipv4;
1213 
1214 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1215 		txbuf->hdrlen = txbuf->tcphlen;
1216 		irdma_ieq_copy_to_txbuf(buf, txbuf, IRDMA_TCP_OFFSET, 0,
1217 					txbuf->hdrlen);
1218 	} else {
1219 		txbuf->maclen = buf->maclen;
1220 		txbuf->hdrlen = buf->hdrlen;
1221 		irdma_ieq_copy_to_txbuf(buf, txbuf, 0, 0, buf->hdrlen);
1222 	}
1223 }
1224 
1225 /**
1226  * irdma_ieq_check_first_buf - check if rcv buffer's seq is in range
1227  * @buf: receive exception buffer
1228  * @fps: first partial sequence number
1229  */
irdma_ieq_check_first_buf(struct irdma_puda_buf * buf,u32 fps)1230 static void irdma_ieq_check_first_buf(struct irdma_puda_buf *buf, u32 fps)
1231 {
1232 	u32 offset;
1233 
1234 	if (buf->seqnum < fps) {
1235 		offset = fps - buf->seqnum;
1236 		if (offset > buf->datalen)
1237 			return;
1238 		buf->data += offset;
1239 		buf->datalen -= (u16)offset;
1240 		buf->seqnum = fps;
1241 	}
1242 }
1243 
1244 /**
1245  * irdma_ieq_compl_pfpdu - write txbuf with full fpdu
1246  * @ieq: ieq resource
1247  * @rxlist: ieq's received buffer list
1248  * @pbufl: temporary list for buffers for fpddu
1249  * @txbuf: tx buffer for fpdu
1250  * @fpdu_len: total length of fpdu
1251  */
irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc * ieq,struct list_head * rxlist,struct list_head * pbufl,struct irdma_puda_buf * txbuf,u16 fpdu_len)1252 static void irdma_ieq_compl_pfpdu(struct irdma_puda_rsrc *ieq,
1253 				  struct list_head *rxlist,
1254 				  struct list_head *pbufl,
1255 				  struct irdma_puda_buf *txbuf, u16 fpdu_len)
1256 {
1257 	struct irdma_puda_buf *buf;
1258 	u32 nextseqnum;
1259 	u16 txoffset, bufoffset;
1260 
1261 	buf = irdma_puda_get_listbuf(pbufl);
1262 	if (!buf)
1263 		return;
1264 
1265 	nextseqnum = buf->seqnum + fpdu_len;
1266 	irdma_ieq_setup_tx_buf(buf, txbuf);
1267 	if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1268 		txoffset = txbuf->hdrlen;
1269 		txbuf->totallen = txbuf->hdrlen + fpdu_len;
1270 		txbuf->data = (u8 *)txbuf->mem.va + txoffset;
1271 	} else {
1272 		txoffset = buf->hdrlen;
1273 		txbuf->totallen = buf->hdrlen + fpdu_len;
1274 		txbuf->data = (u8 *)txbuf->mem.va + buf->hdrlen;
1275 	}
1276 	bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1277 
1278 	do {
1279 		if (buf->datalen >= fpdu_len) {
1280 			/* copied full fpdu */
1281 			irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1282 						fpdu_len);
1283 			buf->datalen -= fpdu_len;
1284 			buf->data += fpdu_len;
1285 			buf->seqnum = nextseqnum;
1286 			break;
1287 		}
1288 		/* copy partial fpdu */
1289 		irdma_ieq_copy_to_txbuf(buf, txbuf, bufoffset, txoffset,
1290 					buf->datalen);
1291 		txoffset += buf->datalen;
1292 		fpdu_len -= buf->datalen;
1293 		irdma_puda_ret_bufpool(ieq, buf);
1294 		buf = irdma_puda_get_listbuf(pbufl);
1295 		if (!buf)
1296 			return;
1297 
1298 		bufoffset = (u16)(buf->data - (u8 *)buf->mem.va);
1299 	} while (1);
1300 
1301 	/* last buffer on the list*/
1302 	if (buf->datalen)
1303 		list_add(&buf->list, rxlist);
1304 	else
1305 		irdma_puda_ret_bufpool(ieq, buf);
1306 }
1307 
1308 /**
1309  * irdma_ieq_create_pbufl - create buffer list for single fpdu
1310  * @pfpdu: pointer to fpdu
1311  * @rxlist: resource list for receive ieq buffes
1312  * @pbufl: temp. list for buffers for fpddu
1313  * @buf: first receive buffer
1314  * @fpdu_len: total length of fpdu
1315  */
irdma_ieq_create_pbufl(struct irdma_pfpdu * pfpdu,struct list_head * rxlist,struct list_head * pbufl,struct irdma_puda_buf * buf,u16 fpdu_len)1316 static int irdma_ieq_create_pbufl(struct irdma_pfpdu *pfpdu,
1317 				  struct list_head *rxlist,
1318 				  struct list_head *pbufl,
1319 				  struct irdma_puda_buf *buf, u16 fpdu_len)
1320 {
1321 	int status = 0;
1322 	struct irdma_puda_buf *nextbuf;
1323 	u32 nextseqnum;
1324 	u16 plen = fpdu_len - buf->datalen;
1325 	bool done = false;
1326 
1327 	nextseqnum = buf->seqnum + buf->datalen;
1328 	do {
1329 		nextbuf = irdma_puda_get_listbuf(rxlist);
1330 		if (!nextbuf) {
1331 			status = -ENOBUFS;
1332 			break;
1333 		}
1334 		list_add_tail(&nextbuf->list, pbufl);
1335 		if (nextbuf->seqnum != nextseqnum) {
1336 			pfpdu->bad_seq_num++;
1337 			status = -ERANGE;
1338 			break;
1339 		}
1340 		if (nextbuf->datalen >= plen) {
1341 			done = true;
1342 		} else {
1343 			plen -= nextbuf->datalen;
1344 			nextseqnum = nextbuf->seqnum + nextbuf->datalen;
1345 		}
1346 
1347 	} while (!done);
1348 
1349 	return status;
1350 }
1351 
1352 /**
1353  * irdma_ieq_handle_partial - process partial fpdu buffer
1354  * @ieq: ieq resource
1355  * @pfpdu: partial management per user qp
1356  * @buf: receive buffer
1357  * @fpdu_len: fpdu len in the buffer
1358  */
irdma_ieq_handle_partial(struct irdma_puda_rsrc * ieq,struct irdma_pfpdu * pfpdu,struct irdma_puda_buf * buf,u16 fpdu_len)1359 static int irdma_ieq_handle_partial(struct irdma_puda_rsrc *ieq,
1360 				    struct irdma_pfpdu *pfpdu,
1361 				    struct irdma_puda_buf *buf, u16 fpdu_len)
1362 {
1363 	int status = 0;
1364 	u8 *crcptr;
1365 	u32 mpacrc;
1366 	u32 seqnum = buf->seqnum;
1367 	struct list_head pbufl; /* partial buffer list */
1368 	struct irdma_puda_buf *txbuf = NULL;
1369 	struct list_head *rxlist = &pfpdu->rxlist;
1370 
1371 	ieq->partials_handled++;
1372 
1373 	INIT_LIST_HEAD(&pbufl);
1374 	list_add(&buf->list, &pbufl);
1375 
1376 	status = irdma_ieq_create_pbufl(pfpdu, rxlist, &pbufl, buf, fpdu_len);
1377 	if (status)
1378 		goto error;
1379 
1380 	txbuf = irdma_puda_get_bufpool(ieq);
1381 	if (!txbuf) {
1382 		pfpdu->no_tx_bufs++;
1383 		status = -ENOBUFS;
1384 		goto error;
1385 	}
1386 
1387 	irdma_ieq_compl_pfpdu(ieq, rxlist, &pbufl, txbuf, fpdu_len);
1388 	irdma_ieq_update_tcpip_info(txbuf, fpdu_len, seqnum);
1389 
1390 	crcptr = txbuf->data + fpdu_len - 4;
1391 	mpacrc = *(u32 *)crcptr;
1392 	if (ieq->check_crc) {
1393 		status = irdma_ieq_check_mpacrc(ieq->hash_desc, txbuf->data,
1394 						(fpdu_len - 4), mpacrc);
1395 		if (status) {
1396 			ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error bad crc\n");
1397 			goto error;
1398 		}
1399 	}
1400 
1401 	print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1402 			     txbuf->mem.va, txbuf->totallen, false);
1403 	if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
1404 		txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1405 	txbuf->do_lpb = true;
1406 	irdma_puda_send_buf(ieq, txbuf);
1407 	pfpdu->rcv_nxt = seqnum + fpdu_len;
1408 	return status;
1409 
1410 error:
1411 	while (!list_empty(&pbufl)) {
1412 		buf = list_last_entry(&pbufl, struct irdma_puda_buf, list);
1413 		list_move(&buf->list, rxlist);
1414 	}
1415 	if (txbuf)
1416 		irdma_puda_ret_bufpool(ieq, txbuf);
1417 
1418 	return status;
1419 }
1420 
1421 /**
1422  * irdma_ieq_process_buf - process buffer rcvd for ieq
1423  * @ieq: ieq resource
1424  * @pfpdu: partial management per user qp
1425  * @buf: receive buffer
1426  */
irdma_ieq_process_buf(struct irdma_puda_rsrc * ieq,struct irdma_pfpdu * pfpdu,struct irdma_puda_buf * buf)1427 static int irdma_ieq_process_buf(struct irdma_puda_rsrc *ieq,
1428 				 struct irdma_pfpdu *pfpdu,
1429 				 struct irdma_puda_buf *buf)
1430 {
1431 	u16 fpdu_len = 0;
1432 	u16 datalen = buf->datalen;
1433 	u8 *datap = buf->data;
1434 	u8 *crcptr;
1435 	u16 ioffset = 0;
1436 	u32 mpacrc;
1437 	u32 seqnum = buf->seqnum;
1438 	u16 len = 0;
1439 	u16 full = 0;
1440 	bool partial = false;
1441 	struct irdma_puda_buf *txbuf;
1442 	struct list_head *rxlist = &pfpdu->rxlist;
1443 	int ret = 0;
1444 
1445 	ioffset = (u16)(buf->data - (u8 *)buf->mem.va);
1446 	while (datalen) {
1447 		fpdu_len = irdma_ieq_get_fpdu_len(pfpdu, datap, buf->seqnum);
1448 		if (!fpdu_len) {
1449 			ibdev_dbg(to_ibdev(ieq->dev),
1450 				  "IEQ: error bad fpdu len\n");
1451 			list_add(&buf->list, rxlist);
1452 			return -EINVAL;
1453 		}
1454 
1455 		if (datalen < fpdu_len) {
1456 			partial = true;
1457 			break;
1458 		}
1459 		crcptr = datap + fpdu_len - 4;
1460 		mpacrc = *(u32 *)crcptr;
1461 		if (ieq->check_crc)
1462 			ret = irdma_ieq_check_mpacrc(ieq->hash_desc, datap,
1463 						     fpdu_len - 4, mpacrc);
1464 		if (ret) {
1465 			list_add(&buf->list, rxlist);
1466 			ibdev_dbg(to_ibdev(ieq->dev),
1467 				  "ERR: IRDMA_ERR_MPA_CRC\n");
1468 			return -EINVAL;
1469 		}
1470 		full++;
1471 		pfpdu->fpdu_processed++;
1472 		ieq->fpdu_processed++;
1473 		datap += fpdu_len;
1474 		len += fpdu_len;
1475 		datalen -= fpdu_len;
1476 	}
1477 	if (full) {
1478 		/* copy full pdu's in the txbuf and send them out */
1479 		txbuf = irdma_puda_get_bufpool(ieq);
1480 		if (!txbuf) {
1481 			pfpdu->no_tx_bufs++;
1482 			list_add(&buf->list, rxlist);
1483 			return -ENOBUFS;
1484 		}
1485 		/* modify txbuf's buffer header */
1486 		irdma_ieq_setup_tx_buf(buf, txbuf);
1487 		/* copy full fpdu's to new buffer */
1488 		if (ieq->dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1489 			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1490 						txbuf->hdrlen, len);
1491 			txbuf->totallen = txbuf->hdrlen + len;
1492 			txbuf->ah_id = pfpdu->ah->ah_info.ah_idx;
1493 		} else {
1494 			irdma_ieq_copy_to_txbuf(buf, txbuf, ioffset,
1495 						buf->hdrlen, len);
1496 			txbuf->totallen = buf->hdrlen + len;
1497 		}
1498 		irdma_ieq_update_tcpip_info(txbuf, len, buf->seqnum);
1499 		print_hex_dump_debug("IEQ: IEQ TX BUFFER", DUMP_PREFIX_OFFSET,
1500 				     16, 8, txbuf->mem.va, txbuf->totallen,
1501 				     false);
1502 		txbuf->do_lpb = true;
1503 		irdma_puda_send_buf(ieq, txbuf);
1504 
1505 		if (!datalen) {
1506 			pfpdu->rcv_nxt = buf->seqnum + len;
1507 			irdma_puda_ret_bufpool(ieq, buf);
1508 			return 0;
1509 		}
1510 		buf->data = datap;
1511 		buf->seqnum = seqnum + len;
1512 		buf->datalen = datalen;
1513 		pfpdu->rcv_nxt = buf->seqnum;
1514 	}
1515 	if (partial)
1516 		return irdma_ieq_handle_partial(ieq, pfpdu, buf, fpdu_len);
1517 
1518 	return 0;
1519 }
1520 
1521 /**
1522  * irdma_ieq_process_fpdus - process fpdu's buffers on its list
1523  * @qp: qp for which partial fpdus
1524  * @ieq: ieq resource
1525  */
irdma_ieq_process_fpdus(struct irdma_sc_qp * qp,struct irdma_puda_rsrc * ieq)1526 void irdma_ieq_process_fpdus(struct irdma_sc_qp *qp,
1527 			     struct irdma_puda_rsrc *ieq)
1528 {
1529 	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1530 	struct list_head *rxlist = &pfpdu->rxlist;
1531 	struct irdma_puda_buf *buf;
1532 	int status;
1533 
1534 	do {
1535 		if (list_empty(rxlist))
1536 			break;
1537 		buf = irdma_puda_get_listbuf(rxlist);
1538 		if (!buf) {
1539 			ibdev_dbg(to_ibdev(ieq->dev), "IEQ: error no buf\n");
1540 			break;
1541 		}
1542 		if (buf->seqnum != pfpdu->rcv_nxt) {
1543 			/* This could be out of order or missing packet */
1544 			pfpdu->out_of_order++;
1545 			list_add(&buf->list, rxlist);
1546 			break;
1547 		}
1548 		/* keep processing buffers from the head of the list */
1549 		status = irdma_ieq_process_buf(ieq, pfpdu, buf);
1550 		if (status == -EINVAL) {
1551 			pfpdu->mpa_crc_err = true;
1552 			while (!list_empty(rxlist)) {
1553 				buf = irdma_puda_get_listbuf(rxlist);
1554 				irdma_puda_ret_bufpool(ieq, buf);
1555 				pfpdu->crc_err++;
1556 				ieq->crc_err++;
1557 			}
1558 			/* create CQP for AE */
1559 			irdma_ieq_mpa_crc_ae(ieq->dev, qp);
1560 		}
1561 	} while (!status);
1562 }
1563 
1564 /**
1565  * irdma_ieq_create_ah - create an address handle for IEQ
1566  * @qp: qp pointer
1567  * @buf: buf received on IEQ used to create AH
1568  */
irdma_ieq_create_ah(struct irdma_sc_qp * qp,struct irdma_puda_buf * buf)1569 static int irdma_ieq_create_ah(struct irdma_sc_qp *qp, struct irdma_puda_buf *buf)
1570 {
1571 	struct irdma_ah_info ah_info = {};
1572 
1573 	qp->pfpdu.ah_buf = buf;
1574 	irdma_puda_ieq_get_ah_info(qp, &ah_info);
1575 	return irdma_puda_create_ah(qp->vsi->dev, &ah_info, false,
1576 				    IRDMA_PUDA_RSRC_TYPE_IEQ, qp,
1577 				    &qp->pfpdu.ah);
1578 }
1579 
1580 /**
1581  * irdma_ieq_handle_exception - handle qp's exception
1582  * @ieq: ieq resource
1583  * @qp: qp receiving excpetion
1584  * @buf: receive buffer
1585  */
irdma_ieq_handle_exception(struct irdma_puda_rsrc * ieq,struct irdma_sc_qp * qp,struct irdma_puda_buf * buf)1586 static void irdma_ieq_handle_exception(struct irdma_puda_rsrc *ieq,
1587 				       struct irdma_sc_qp *qp,
1588 				       struct irdma_puda_buf *buf)
1589 {
1590 	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1591 	u32 *hw_host_ctx = (u32 *)qp->hw_host_ctx;
1592 	u32 rcv_wnd = hw_host_ctx[23];
1593 	/* first partial seq # in q2 */
1594 	u32 fps = *(u32 *)(qp->q2_buf + Q2_FPSN_OFFSET);
1595 	struct list_head *rxlist = &pfpdu->rxlist;
1596 	unsigned long flags = 0;
1597 	u8 hw_rev = qp->dev->hw_attrs.uk_attrs.hw_rev;
1598 
1599 	print_hex_dump_debug("IEQ: IEQ RX BUFFER", DUMP_PREFIX_OFFSET, 16, 8,
1600 			     buf->mem.va, buf->totallen, false);
1601 
1602 	spin_lock_irqsave(&pfpdu->lock, flags);
1603 	pfpdu->total_ieq_bufs++;
1604 	if (pfpdu->mpa_crc_err) {
1605 		pfpdu->crc_err++;
1606 		goto error;
1607 	}
1608 	if (pfpdu->mode && fps != pfpdu->fps) {
1609 		/* clean up qp as it is new partial sequence */
1610 		irdma_ieq_cleanup_qp(ieq, qp);
1611 		ibdev_dbg(to_ibdev(ieq->dev), "IEQ: restarting new partial\n");
1612 		pfpdu->mode = false;
1613 	}
1614 
1615 	if (!pfpdu->mode) {
1616 		print_hex_dump_debug("IEQ: Q2 BUFFER", DUMP_PREFIX_OFFSET, 16,
1617 				     8, (u64 *)qp->q2_buf, 128, false);
1618 		/* First_Partial_Sequence_Number check */
1619 		pfpdu->rcv_nxt = fps;
1620 		pfpdu->fps = fps;
1621 		pfpdu->mode = true;
1622 		pfpdu->max_fpdu_data = (buf->ipv4) ?
1623 				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV4) :
1624 				       (ieq->vsi->mtu - IRDMA_MTU_TO_MSS_IPV6);
1625 		pfpdu->pmode_count++;
1626 		ieq->pmode_count++;
1627 		INIT_LIST_HEAD(rxlist);
1628 		irdma_ieq_check_first_buf(buf, fps);
1629 	}
1630 
1631 	if (!(rcv_wnd >= (buf->seqnum - pfpdu->rcv_nxt))) {
1632 		pfpdu->bad_seq_num++;
1633 		ieq->bad_seq_num++;
1634 		goto error;
1635 	}
1636 
1637 	if (!list_empty(rxlist)) {
1638 		if (buf->seqnum != pfpdu->nextseqnum) {
1639 			irdma_send_ieq_ack(qp);
1640 			/* throw away out-of-order, duplicates*/
1641 			goto error;
1642 		}
1643 	}
1644 	/* Insert buf before head */
1645 	list_add_tail(&buf->list, rxlist);
1646 	pfpdu->nextseqnum = buf->seqnum + buf->datalen;
1647 	pfpdu->lastrcv_buf = buf;
1648 	if (hw_rev >= IRDMA_GEN_2 && !pfpdu->ah) {
1649 		irdma_ieq_create_ah(qp, buf);
1650 		if (!pfpdu->ah)
1651 			goto error;
1652 		goto exit;
1653 	}
1654 	if (hw_rev == IRDMA_GEN_1)
1655 		irdma_ieq_process_fpdus(qp, ieq);
1656 	else if (pfpdu->ah && pfpdu->ah->ah_info.ah_valid)
1657 		irdma_ieq_process_fpdus(qp, ieq);
1658 exit:
1659 	spin_unlock_irqrestore(&pfpdu->lock, flags);
1660 
1661 	return;
1662 
1663 error:
1664 	irdma_puda_ret_bufpool(ieq, buf);
1665 	spin_unlock_irqrestore(&pfpdu->lock, flags);
1666 }
1667 
1668 /**
1669  * irdma_ieq_receive - received exception buffer
1670  * @vsi: VSI of device
1671  * @buf: exception buffer received
1672  */
irdma_ieq_receive(struct irdma_sc_vsi * vsi,struct irdma_puda_buf * buf)1673 static void irdma_ieq_receive(struct irdma_sc_vsi *vsi,
1674 			      struct irdma_puda_buf *buf)
1675 {
1676 	struct irdma_puda_rsrc *ieq = vsi->ieq;
1677 	struct irdma_sc_qp *qp = NULL;
1678 	u32 wqe_idx = ieq->compl_rxwqe_idx;
1679 
1680 	qp = irdma_ieq_get_qp(vsi->dev, buf);
1681 	if (!qp) {
1682 		ieq->stats_bad_qp_id++;
1683 		irdma_puda_ret_bufpool(ieq, buf);
1684 	} else {
1685 		irdma_ieq_handle_exception(ieq, qp, buf);
1686 	}
1687 	/*
1688 	 * ieq->rx_wqe_idx is used by irdma_puda_replenish_rq()
1689 	 * on which wqe_idx to start replenish rq
1690 	 */
1691 	if (!ieq->rxq_invalid_cnt)
1692 		ieq->rx_wqe_idx = wqe_idx;
1693 	ieq->rxq_invalid_cnt++;
1694 }
1695 
1696 /**
1697  * irdma_ieq_tx_compl - put back after sending completed exception buffer
1698  * @vsi: sc VSI struct
1699  * @sqwrid: pointer to puda buffer
1700  */
irdma_ieq_tx_compl(struct irdma_sc_vsi * vsi,void * sqwrid)1701 static void irdma_ieq_tx_compl(struct irdma_sc_vsi *vsi, void *sqwrid)
1702 {
1703 	struct irdma_puda_rsrc *ieq = vsi->ieq;
1704 	struct irdma_puda_buf *buf = sqwrid;
1705 
1706 	irdma_puda_ret_bufpool(ieq, buf);
1707 }
1708 
1709 /**
1710  * irdma_ieq_cleanup_qp - qp is being destroyed
1711  * @ieq: ieq resource
1712  * @qp: all pending fpdu buffers
1713  */
irdma_ieq_cleanup_qp(struct irdma_puda_rsrc * ieq,struct irdma_sc_qp * qp)1714 void irdma_ieq_cleanup_qp(struct irdma_puda_rsrc *ieq, struct irdma_sc_qp *qp)
1715 {
1716 	struct irdma_puda_buf *buf;
1717 	struct irdma_pfpdu *pfpdu = &qp->pfpdu;
1718 	struct list_head *rxlist = &pfpdu->rxlist;
1719 
1720 	if (qp->pfpdu.ah) {
1721 		irdma_puda_free_ah(ieq->dev, qp->pfpdu.ah);
1722 		qp->pfpdu.ah = NULL;
1723 		qp->pfpdu.ah_buf = NULL;
1724 	}
1725 
1726 	if (!pfpdu->mode)
1727 		return;
1728 
1729 	while (!list_empty(rxlist)) {
1730 		buf = irdma_puda_get_listbuf(rxlist);
1731 		irdma_puda_ret_bufpool(ieq, buf);
1732 	}
1733 }
1734