1 // SPDX-License-Identifier: GPL-2.0
2 /* Marvell RVU Ethernet driver
3 *
4 * Copyright (C) 2020 Marvell.
5 *
6 */
7
8 #include <linux/etherdevice.h>
9 #include <net/ip.h>
10 #include <net/tso.h>
11 #include <linux/bpf.h>
12 #include <linux/bpf_trace.h>
13
14 #include "otx2_reg.h"
15 #include "otx2_common.h"
16 #include "otx2_struct.h"
17 #include "otx2_txrx.h"
18 #include "otx2_ptp.h"
19 #include "cn10k.h"
20
21 #define CQE_ADDR(CQ, idx) ((CQ)->cqe_base + ((CQ)->cqe_size * (idx)))
22 #define PTP_PORT 0x13F
23 /* PTPv2 header Original Timestamp starts at byte offset 34 and
24 * contains 6 byte seconds field and 4 byte nano seconds field.
25 */
26 #define PTP_SYNC_SEC_OFFSET 34
27
28 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
29 struct bpf_prog *prog,
30 struct nix_cqe_rx_s *cqe,
31 struct otx2_cq_queue *cq);
32
otx2_nix_cq_op_status(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)33 static int otx2_nix_cq_op_status(struct otx2_nic *pfvf,
34 struct otx2_cq_queue *cq)
35 {
36 u64 incr = (u64)(cq->cq_idx) << 32;
37 u64 status;
38
39 status = otx2_atomic64_fetch_add(incr, pfvf->cq_op_addr);
40
41 if (unlikely(status & BIT_ULL(CQ_OP_STAT_OP_ERR) ||
42 status & BIT_ULL(CQ_OP_STAT_CQ_ERR))) {
43 dev_err(pfvf->dev, "CQ stopped due to error");
44 return -EINVAL;
45 }
46
47 cq->cq_tail = status & 0xFFFFF;
48 cq->cq_head = (status >> 20) & 0xFFFFF;
49 if (cq->cq_tail < cq->cq_head)
50 cq->pend_cqe = (cq->cqe_cnt - cq->cq_head) +
51 cq->cq_tail;
52 else
53 cq->pend_cqe = cq->cq_tail - cq->cq_head;
54
55 return 0;
56 }
57
otx2_get_next_cqe(struct otx2_cq_queue * cq)58 static struct nix_cqe_hdr_s *otx2_get_next_cqe(struct otx2_cq_queue *cq)
59 {
60 struct nix_cqe_hdr_s *cqe_hdr;
61
62 cqe_hdr = (struct nix_cqe_hdr_s *)CQE_ADDR(cq, cq->cq_head);
63 if (cqe_hdr->cqe_type == NIX_XQE_TYPE_INVALID)
64 return NULL;
65
66 cq->cq_head++;
67 cq->cq_head &= (cq->cqe_cnt - 1);
68
69 return cqe_hdr;
70 }
71
frag_num(unsigned int i)72 static unsigned int frag_num(unsigned int i)
73 {
74 #ifdef __BIG_ENDIAN
75 return (i & ~3) + 3 - (i & 3);
76 #else
77 return i;
78 #endif
79 }
80
otx2_dma_map_skb_frag(struct otx2_nic * pfvf,struct sk_buff * skb,int seg,int * len)81 static dma_addr_t otx2_dma_map_skb_frag(struct otx2_nic *pfvf,
82 struct sk_buff *skb, int seg, int *len)
83 {
84 const skb_frag_t *frag;
85 struct page *page;
86 int offset;
87
88 /* First segment is always skb->data */
89 if (!seg) {
90 page = virt_to_page(skb->data);
91 offset = offset_in_page(skb->data);
92 *len = skb_headlen(skb);
93 } else {
94 frag = &skb_shinfo(skb)->frags[seg - 1];
95 page = skb_frag_page(frag);
96 offset = skb_frag_off(frag);
97 *len = skb_frag_size(frag);
98 }
99 return otx2_dma_map_page(pfvf, page, offset, *len, DMA_TO_DEVICE);
100 }
101
otx2_dma_unmap_skb_frags(struct otx2_nic * pfvf,struct sg_list * sg)102 static void otx2_dma_unmap_skb_frags(struct otx2_nic *pfvf, struct sg_list *sg)
103 {
104 int seg;
105
106 for (seg = 0; seg < sg->num_segs; seg++) {
107 otx2_dma_unmap_page(pfvf, sg->dma_addr[seg],
108 sg->size[seg], DMA_TO_DEVICE);
109 }
110 sg->num_segs = 0;
111 }
112
otx2_xdp_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe)113 static void otx2_xdp_snd_pkt_handler(struct otx2_nic *pfvf,
114 struct otx2_snd_queue *sq,
115 struct nix_cqe_tx_s *cqe)
116 {
117 struct nix_send_comp_s *snd_comp = &cqe->comp;
118 struct sg_list *sg;
119 struct page *page;
120 u64 pa;
121
122 sg = &sq->sg[snd_comp->sqe_id];
123
124 pa = otx2_iova_to_phys(pfvf->iommu_domain, sg->dma_addr[0]);
125 otx2_dma_unmap_page(pfvf, sg->dma_addr[0],
126 sg->size[0], DMA_TO_DEVICE);
127 page = virt_to_page(phys_to_virt(pa));
128 put_page(page);
129 }
130
otx2_snd_pkt_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,struct otx2_snd_queue * sq,struct nix_cqe_tx_s * cqe,int budget,int * tx_pkts,int * tx_bytes)131 static void otx2_snd_pkt_handler(struct otx2_nic *pfvf,
132 struct otx2_cq_queue *cq,
133 struct otx2_snd_queue *sq,
134 struct nix_cqe_tx_s *cqe,
135 int budget, int *tx_pkts, int *tx_bytes)
136 {
137 struct nix_send_comp_s *snd_comp = &cqe->comp;
138 struct skb_shared_hwtstamps ts;
139 struct sk_buff *skb = NULL;
140 u64 timestamp, tsns;
141 struct sg_list *sg;
142 int err;
143
144 if (unlikely(snd_comp->status) && netif_msg_tx_err(pfvf))
145 net_err_ratelimited("%s: TX%d: Error in send CQ status:%x\n",
146 pfvf->netdev->name, cq->cint_idx,
147 snd_comp->status);
148
149 sg = &sq->sg[snd_comp->sqe_id];
150 skb = (struct sk_buff *)sg->skb;
151 if (unlikely(!skb))
152 return;
153
154 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
155 timestamp = ((u64 *)sq->timestamps->base)[snd_comp->sqe_id];
156 if (timestamp != 1) {
157 timestamp = pfvf->ptp->convert_tx_ptp_tstmp(timestamp);
158 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
159 if (!err) {
160 memset(&ts, 0, sizeof(ts));
161 ts.hwtstamp = ns_to_ktime(tsns);
162 skb_tstamp_tx(skb, &ts);
163 }
164 }
165 }
166
167 *tx_bytes += skb->len;
168 (*tx_pkts)++;
169 otx2_dma_unmap_skb_frags(pfvf, sg);
170 napi_consume_skb(skb, budget);
171 sg->skb = (u64)NULL;
172 }
173
otx2_set_rxtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,void * data)174 static void otx2_set_rxtstamp(struct otx2_nic *pfvf,
175 struct sk_buff *skb, void *data)
176 {
177 u64 timestamp, tsns;
178 int err;
179
180 if (!(pfvf->flags & OTX2_FLAG_RX_TSTAMP_ENABLED))
181 return;
182
183 timestamp = pfvf->ptp->convert_rx_ptp_tstmp(*(u64 *)data);
184 /* The first 8 bytes is the timestamp */
185 err = otx2_ptp_tstamp2time(pfvf, timestamp, &tsns);
186 if (err)
187 return;
188
189 skb_hwtstamps(skb)->hwtstamp = ns_to_ktime(tsns);
190 }
191
otx2_skb_add_frag(struct otx2_nic * pfvf,struct sk_buff * skb,u64 iova,int len,struct nix_rx_parse_s * parse,int qidx)192 static bool otx2_skb_add_frag(struct otx2_nic *pfvf, struct sk_buff *skb,
193 u64 iova, int len, struct nix_rx_parse_s *parse,
194 int qidx)
195 {
196 struct page *page;
197 int off = 0;
198 void *va;
199
200 va = phys_to_virt(otx2_iova_to_phys(pfvf->iommu_domain, iova));
201
202 if (likely(!skb_shinfo(skb)->nr_frags)) {
203 /* Check if data starts at some nonzero offset
204 * from the start of the buffer. For now the
205 * only possible offset is 8 bytes in the case
206 * where packet is prepended by a timestamp.
207 */
208 if (parse->laptr) {
209 otx2_set_rxtstamp(pfvf, skb, va);
210 off = OTX2_HW_TIMESTAMP_LEN;
211 }
212 }
213
214 page = virt_to_page(va);
215 if (likely(skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)) {
216 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
217 va - page_address(page) + off,
218 len - off, pfvf->rbsize);
219
220 otx2_dma_unmap_page(pfvf, iova - OTX2_HEAD_ROOM,
221 pfvf->rbsize, DMA_FROM_DEVICE);
222 return true;
223 }
224
225 /* If more than MAX_SKB_FRAGS fragments are received then
226 * give back those buffer pointers to hardware for reuse.
227 */
228 pfvf->hw_ops->aura_freeptr(pfvf, qidx, iova & ~0x07ULL);
229
230 return false;
231 }
232
otx2_set_rxhash(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,struct sk_buff * skb)233 static void otx2_set_rxhash(struct otx2_nic *pfvf,
234 struct nix_cqe_rx_s *cqe, struct sk_buff *skb)
235 {
236 enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
237 struct otx2_rss_info *rss;
238 u32 hash = 0;
239
240 if (!(pfvf->netdev->features & NETIF_F_RXHASH))
241 return;
242
243 rss = &pfvf->hw.rss_info;
244 if (rss->flowkey_cfg) {
245 if (rss->flowkey_cfg &
246 ~(NIX_FLOW_KEY_TYPE_IPV4 | NIX_FLOW_KEY_TYPE_IPV6))
247 hash_type = PKT_HASH_TYPE_L4;
248 else
249 hash_type = PKT_HASH_TYPE_L3;
250 hash = cqe->hdr.flow_tag;
251 }
252 skb_set_hash(skb, hash, hash_type);
253 }
254
otx2_free_rcv_seg(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)255 static void otx2_free_rcv_seg(struct otx2_nic *pfvf, struct nix_cqe_rx_s *cqe,
256 int qidx)
257 {
258 struct nix_rx_sg_s *sg = &cqe->sg;
259 void *end, *start;
260 u64 *seg_addr;
261 int seg;
262
263 start = (void *)sg;
264 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
265 while (start < end) {
266 sg = (struct nix_rx_sg_s *)start;
267 seg_addr = &sg->seg_addr;
268 for (seg = 0; seg < sg->segs; seg++, seg_addr++)
269 pfvf->hw_ops->aura_freeptr(pfvf, qidx,
270 *seg_addr & ~0x07ULL);
271 start += sizeof(*sg);
272 }
273 }
274
otx2_check_rcv_errors(struct otx2_nic * pfvf,struct nix_cqe_rx_s * cqe,int qidx)275 static bool otx2_check_rcv_errors(struct otx2_nic *pfvf,
276 struct nix_cqe_rx_s *cqe, int qidx)
277 {
278 struct otx2_drv_stats *stats = &pfvf->hw.drv_stats;
279 struct nix_rx_parse_s *parse = &cqe->parse;
280
281 if (netif_msg_rx_err(pfvf))
282 netdev_err(pfvf->netdev,
283 "RQ%d: Error pkt with errlev:0x%x errcode:0x%x\n",
284 qidx, parse->errlev, parse->errcode);
285
286 if (parse->errlev == NPC_ERRLVL_RE) {
287 switch (parse->errcode) {
288 case ERRCODE_FCS:
289 case ERRCODE_FCS_RCV:
290 atomic_inc(&stats->rx_fcs_errs);
291 break;
292 case ERRCODE_UNDERSIZE:
293 atomic_inc(&stats->rx_undersize_errs);
294 break;
295 case ERRCODE_OVERSIZE:
296 atomic_inc(&stats->rx_oversize_errs);
297 break;
298 case ERRCODE_OL2_LEN_MISMATCH:
299 atomic_inc(&stats->rx_len_errs);
300 break;
301 default:
302 atomic_inc(&stats->rx_other_errs);
303 break;
304 }
305 } else if (parse->errlev == NPC_ERRLVL_NIX) {
306 switch (parse->errcode) {
307 case ERRCODE_OL3_LEN:
308 case ERRCODE_OL4_LEN:
309 case ERRCODE_IL3_LEN:
310 case ERRCODE_IL4_LEN:
311 atomic_inc(&stats->rx_len_errs);
312 break;
313 case ERRCODE_OL4_CSUM:
314 case ERRCODE_IL4_CSUM:
315 atomic_inc(&stats->rx_csum_errs);
316 break;
317 default:
318 atomic_inc(&stats->rx_other_errs);
319 break;
320 }
321 } else {
322 atomic_inc(&stats->rx_other_errs);
323 /* For now ignore all the NPC parser errors and
324 * pass the packets to stack.
325 */
326 return false;
327 }
328
329 /* If RXALL is enabled pass on packets to stack. */
330 if (pfvf->netdev->features & NETIF_F_RXALL)
331 return false;
332
333 /* Free buffer back to pool */
334 if (cqe->sg.segs)
335 otx2_free_rcv_seg(pfvf, cqe, qidx);
336 return true;
337 }
338
otx2_rcv_pkt_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,struct nix_cqe_rx_s * cqe)339 static void otx2_rcv_pkt_handler(struct otx2_nic *pfvf,
340 struct napi_struct *napi,
341 struct otx2_cq_queue *cq,
342 struct nix_cqe_rx_s *cqe)
343 {
344 struct nix_rx_parse_s *parse = &cqe->parse;
345 struct nix_rx_sg_s *sg = &cqe->sg;
346 struct sk_buff *skb = NULL;
347 void *end, *start;
348 u64 *seg_addr;
349 u16 *seg_size;
350 int seg;
351
352 if (unlikely(parse->errlev || parse->errcode)) {
353 if (otx2_check_rcv_errors(pfvf, cqe, cq->cq_idx))
354 return;
355 }
356
357 if (pfvf->xdp_prog)
358 if (otx2_xdp_rcv_pkt_handler(pfvf, pfvf->xdp_prog, cqe, cq))
359 return;
360
361 skb = napi_get_frags(napi);
362 if (unlikely(!skb))
363 return;
364
365 start = (void *)sg;
366 end = start + ((cqe->parse.desc_sizem1 + 1) * 16);
367 while (start < end) {
368 sg = (struct nix_rx_sg_s *)start;
369 seg_addr = &sg->seg_addr;
370 seg_size = (void *)sg;
371 for (seg = 0; seg < sg->segs; seg++, seg_addr++) {
372 if (otx2_skb_add_frag(pfvf, skb, *seg_addr,
373 seg_size[seg], parse, cq->cq_idx))
374 cq->pool_ptrs++;
375 }
376 start += sizeof(*sg);
377 }
378 otx2_set_rxhash(pfvf, cqe, skb);
379
380 skb_record_rx_queue(skb, cq->cq_idx);
381 if (pfvf->netdev->features & NETIF_F_RXCSUM)
382 skb->ip_summed = CHECKSUM_UNNECESSARY;
383
384 napi_gro_frags(napi);
385 }
386
otx2_rx_napi_handler(struct otx2_nic * pfvf,struct napi_struct * napi,struct otx2_cq_queue * cq,int budget)387 static int otx2_rx_napi_handler(struct otx2_nic *pfvf,
388 struct napi_struct *napi,
389 struct otx2_cq_queue *cq, int budget)
390 {
391 struct nix_cqe_rx_s *cqe;
392 int processed_cqe = 0;
393
394 if (cq->pend_cqe >= budget)
395 goto process_cqe;
396
397 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
398 return 0;
399
400 process_cqe:
401 while (likely(processed_cqe < budget) && cq->pend_cqe) {
402 cqe = (struct nix_cqe_rx_s *)CQE_ADDR(cq, cq->cq_head);
403 if (cqe->hdr.cqe_type == NIX_XQE_TYPE_INVALID ||
404 !cqe->sg.seg_addr) {
405 if (!processed_cqe)
406 return 0;
407 break;
408 }
409 cq->cq_head++;
410 cq->cq_head &= (cq->cqe_cnt - 1);
411
412 otx2_rcv_pkt_handler(pfvf, napi, cq, cqe);
413
414 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
415 cqe->sg.seg_addr = 0x00;
416 processed_cqe++;
417 cq->pend_cqe--;
418 }
419
420 /* Free CQEs to HW */
421 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
422 ((u64)cq->cq_idx << 32) | processed_cqe);
423
424 return processed_cqe;
425 }
426
otx2_refill_pool_ptrs(void * dev,struct otx2_cq_queue * cq)427 void otx2_refill_pool_ptrs(void *dev, struct otx2_cq_queue *cq)
428 {
429 struct otx2_nic *pfvf = dev;
430 dma_addr_t bufptr;
431
432 while (cq->pool_ptrs) {
433 if (otx2_alloc_buffer(pfvf, cq, &bufptr))
434 break;
435 otx2_aura_freeptr(pfvf, cq->cq_idx, bufptr + OTX2_HEAD_ROOM);
436 cq->pool_ptrs--;
437 }
438 }
439
otx2_tx_napi_handler(struct otx2_nic * pfvf,struct otx2_cq_queue * cq,int budget)440 static int otx2_tx_napi_handler(struct otx2_nic *pfvf,
441 struct otx2_cq_queue *cq, int budget)
442 {
443 int tx_pkts = 0, tx_bytes = 0, qidx;
444 struct otx2_snd_queue *sq;
445 struct nix_cqe_tx_s *cqe;
446 int processed_cqe = 0;
447
448 if (cq->pend_cqe >= budget)
449 goto process_cqe;
450
451 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
452 return 0;
453
454 process_cqe:
455 qidx = cq->cq_idx - pfvf->hw.rx_queues;
456 sq = &pfvf->qset.sq[qidx];
457
458 while (likely(processed_cqe < budget) && cq->pend_cqe) {
459 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
460 if (unlikely(!cqe)) {
461 if (!processed_cqe)
462 return 0;
463 break;
464 }
465
466 if (cq->cq_type == CQ_XDP) {
467 otx2_xdp_snd_pkt_handler(pfvf, sq, cqe);
468 } else {
469 otx2_snd_pkt_handler(pfvf, cq, sq, cqe, budget,
470 &tx_pkts, &tx_bytes);
471 }
472
473 cqe->hdr.cqe_type = NIX_XQE_TYPE_INVALID;
474 processed_cqe++;
475 cq->pend_cqe--;
476
477 sq->cons_head++;
478 sq->cons_head &= (sq->sqe_cnt - 1);
479 }
480
481 /* Free CQEs to HW */
482 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
483 ((u64)cq->cq_idx << 32) | processed_cqe);
484
485 if (likely(tx_pkts)) {
486 struct netdev_queue *txq;
487
488 txq = netdev_get_tx_queue(pfvf->netdev, cq->cint_idx);
489 netdev_tx_completed_queue(txq, tx_pkts, tx_bytes);
490 /* Check if queue was stopped earlier due to ring full */
491 smp_mb();
492 if (netif_tx_queue_stopped(txq) &&
493 netif_carrier_ok(pfvf->netdev))
494 netif_tx_wake_queue(txq);
495 }
496 return 0;
497 }
498
otx2_adjust_adaptive_coalese(struct otx2_nic * pfvf,struct otx2_cq_poll * cq_poll)499 static void otx2_adjust_adaptive_coalese(struct otx2_nic *pfvf, struct otx2_cq_poll *cq_poll)
500 {
501 struct dim_sample dim_sample;
502 u64 rx_frames, rx_bytes;
503
504 rx_frames = OTX2_GET_RX_STATS(RX_BCAST) + OTX2_GET_RX_STATS(RX_MCAST) +
505 OTX2_GET_RX_STATS(RX_UCAST);
506 rx_bytes = OTX2_GET_RX_STATS(RX_OCTS);
507 dim_update_sample(pfvf->napi_events, rx_frames, rx_bytes, &dim_sample);
508 net_dim(&cq_poll->dim, dim_sample);
509 }
510
otx2_napi_handler(struct napi_struct * napi,int budget)511 int otx2_napi_handler(struct napi_struct *napi, int budget)
512 {
513 struct otx2_cq_queue *rx_cq = NULL;
514 struct otx2_cq_poll *cq_poll;
515 int workdone = 0, cq_idx, i;
516 struct otx2_cq_queue *cq;
517 struct otx2_qset *qset;
518 struct otx2_nic *pfvf;
519
520 cq_poll = container_of(napi, struct otx2_cq_poll, napi);
521 pfvf = (struct otx2_nic *)cq_poll->dev;
522 qset = &pfvf->qset;
523
524 for (i = 0; i < CQS_PER_CINT; i++) {
525 cq_idx = cq_poll->cq_ids[i];
526 if (unlikely(cq_idx == CINT_INVALID_CQ))
527 continue;
528 cq = &qset->cq[cq_idx];
529 if (cq->cq_type == CQ_RX) {
530 rx_cq = cq;
531 workdone += otx2_rx_napi_handler(pfvf, napi,
532 cq, budget);
533 } else {
534 workdone += otx2_tx_napi_handler(pfvf, cq, budget);
535 }
536 }
537
538 if (rx_cq && rx_cq->pool_ptrs)
539 pfvf->hw_ops->refill_pool_ptrs(pfvf, rx_cq);
540 /* Clear the IRQ */
541 otx2_write64(pfvf, NIX_LF_CINTX_INT(cq_poll->cint_idx), BIT_ULL(0));
542
543 if (workdone < budget && napi_complete_done(napi, workdone)) {
544 /* If interface is going down, don't re-enable IRQ */
545 if (pfvf->flags & OTX2_FLAG_INTF_DOWN)
546 return workdone;
547
548 /* Check for adaptive interrupt coalesce */
549 if (workdone != 0 &&
550 ((pfvf->flags & OTX2_FLAG_ADPTV_INT_COAL_ENABLED) ==
551 OTX2_FLAG_ADPTV_INT_COAL_ENABLED)) {
552 /* Adjust irq coalese using net_dim */
553 otx2_adjust_adaptive_coalese(pfvf, cq_poll);
554 /* Update irq coalescing */
555 for (i = 0; i < pfvf->hw.cint_cnt; i++)
556 otx2_config_irq_coalescing(pfvf, i);
557 }
558
559 /* Re-enable interrupts */
560 otx2_write64(pfvf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx),
561 BIT_ULL(0));
562 }
563 return workdone;
564 }
565
otx2_sqe_flush(void * dev,struct otx2_snd_queue * sq,int size,int qidx)566 void otx2_sqe_flush(void *dev, struct otx2_snd_queue *sq,
567 int size, int qidx)
568 {
569 u64 status;
570
571 /* Packet data stores should finish before SQE is flushed to HW */
572 dma_wmb();
573
574 do {
575 memcpy(sq->lmt_addr, sq->sqe_base, size);
576 status = otx2_lmt_flush(sq->io_addr);
577 } while (status == 0);
578
579 sq->head++;
580 sq->head &= (sq->sqe_cnt - 1);
581 }
582
583 #define MAX_SEGS_PER_SG 3
584 /* Add SQE scatter/gather subdescriptor structure */
otx2_sqe_add_sg(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int num_segs,int * offset)585 static bool otx2_sqe_add_sg(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
586 struct sk_buff *skb, int num_segs, int *offset)
587 {
588 struct nix_sqe_sg_s *sg = NULL;
589 u64 dma_addr, *iova = NULL;
590 u16 *sg_lens = NULL;
591 int seg, len;
592
593 sq->sg[sq->head].num_segs = 0;
594
595 for (seg = 0; seg < num_segs; seg++) {
596 if ((seg % MAX_SEGS_PER_SG) == 0) {
597 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
598 sg->ld_type = NIX_SEND_LDTYPE_LDD;
599 sg->subdc = NIX_SUBDC_SG;
600 sg->segs = 0;
601 sg_lens = (void *)sg;
602 iova = (void *)sg + sizeof(*sg);
603 /* Next subdc always starts at a 16byte boundary.
604 * So if sg->segs is whether 2 or 3, offset += 16bytes.
605 */
606 if ((num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
607 *offset += sizeof(*sg) + (3 * sizeof(u64));
608 else
609 *offset += sizeof(*sg) + sizeof(u64);
610 }
611 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
612 if (dma_mapping_error(pfvf->dev, dma_addr))
613 return false;
614
615 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = len;
616 sg->segs++;
617 *iova++ = dma_addr;
618
619 /* Save DMA mapping info for later unmapping */
620 sq->sg[sq->head].dma_addr[seg] = dma_addr;
621 sq->sg[sq->head].size[seg] = len;
622 sq->sg[sq->head].num_segs++;
623 }
624
625 sq->sg[sq->head].skb = (u64)skb;
626 return true;
627 }
628
629 /* Add SQE extended header subdescriptor */
otx2_sqe_add_ext(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int * offset)630 static void otx2_sqe_add_ext(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
631 struct sk_buff *skb, int *offset)
632 {
633 struct nix_sqe_ext_s *ext;
634
635 ext = (struct nix_sqe_ext_s *)(sq->sqe_base + *offset);
636 ext->subdc = NIX_SUBDC_EXT;
637 if (skb_shinfo(skb)->gso_size) {
638 ext->lso = 1;
639 ext->lso_sb = skb_tcp_all_headers(skb);
640 ext->lso_mps = skb_shinfo(skb)->gso_size;
641
642 /* Only TSOv4 and TSOv6 GSO offloads are supported */
643 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) {
644 ext->lso_format = pfvf->hw.lso_tsov4_idx;
645
646 /* HW adds payload size to 'ip_hdr->tot_len' while
647 * sending TSO segment, hence set payload length
648 * in IP header of the packet to just header length.
649 */
650 ip_hdr(skb)->tot_len =
651 htons(ext->lso_sb - skb_network_offset(skb));
652 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) {
653 ext->lso_format = pfvf->hw.lso_tsov6_idx;
654
655 ipv6_hdr(skb)->payload_len =
656 htons(ext->lso_sb - skb_network_offset(skb));
657 } else if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
658 __be16 l3_proto = vlan_get_protocol(skb);
659 struct udphdr *udph = udp_hdr(skb);
660 u16 iplen;
661
662 ext->lso_sb = skb_transport_offset(skb) +
663 sizeof(struct udphdr);
664
665 /* HW adds payload size to length fields in IP and
666 * UDP headers while segmentation, hence adjust the
667 * lengths to just header sizes.
668 */
669 iplen = htons(ext->lso_sb - skb_network_offset(skb));
670 if (l3_proto == htons(ETH_P_IP)) {
671 ip_hdr(skb)->tot_len = iplen;
672 ext->lso_format = pfvf->hw.lso_udpv4_idx;
673 } else {
674 ipv6_hdr(skb)->payload_len = iplen;
675 ext->lso_format = pfvf->hw.lso_udpv6_idx;
676 }
677
678 udph->len = htons(sizeof(struct udphdr));
679 }
680 } else if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
681 ext->tstmp = 1;
682 }
683
684 #define OTX2_VLAN_PTR_OFFSET (ETH_HLEN - ETH_TLEN)
685 if (skb_vlan_tag_present(skb)) {
686 if (skb->vlan_proto == htons(ETH_P_8021Q)) {
687 ext->vlan1_ins_ena = 1;
688 ext->vlan1_ins_ptr = OTX2_VLAN_PTR_OFFSET;
689 ext->vlan1_ins_tci = skb_vlan_tag_get(skb);
690 } else if (skb->vlan_proto == htons(ETH_P_8021AD)) {
691 ext->vlan0_ins_ena = 1;
692 ext->vlan0_ins_ptr = OTX2_VLAN_PTR_OFFSET;
693 ext->vlan0_ins_tci = skb_vlan_tag_get(skb);
694 }
695 }
696
697 *offset += sizeof(*ext);
698 }
699
otx2_sqe_add_mem(struct otx2_snd_queue * sq,int * offset,int alg,u64 iova,int ptp_offset,u64 base_ns,int udp_csum)700 static void otx2_sqe_add_mem(struct otx2_snd_queue *sq, int *offset,
701 int alg, u64 iova, int ptp_offset,
702 u64 base_ns, int udp_csum)
703 {
704 struct nix_sqe_mem_s *mem;
705
706 mem = (struct nix_sqe_mem_s *)(sq->sqe_base + *offset);
707 mem->subdc = NIX_SUBDC_MEM;
708 mem->alg = alg;
709 mem->wmem = 1; /* wait for the memory operation */
710 mem->addr = iova;
711
712 if (ptp_offset) {
713 mem->start_offset = ptp_offset;
714 mem->udp_csum_crt = udp_csum;
715 mem->base_ns = base_ns;
716 mem->step_type = 1;
717 }
718
719 *offset += sizeof(*mem);
720 }
721
722 /* Add SQE header subdescriptor structure */
otx2_sqe_add_hdr(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct nix_sqe_hdr_s * sqe_hdr,struct sk_buff * skb,u16 qidx)723 static void otx2_sqe_add_hdr(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
724 struct nix_sqe_hdr_s *sqe_hdr,
725 struct sk_buff *skb, u16 qidx)
726 {
727 int proto = 0;
728
729 /* Check if SQE was framed before, if yes then no need to
730 * set these constants again and again.
731 */
732 if (!sqe_hdr->total) {
733 /* Don't free Tx buffers to Aura */
734 sqe_hdr->df = 1;
735 sqe_hdr->aura = sq->aura_id;
736 /* Post a CQE Tx after pkt transmission */
737 sqe_hdr->pnc = 1;
738 sqe_hdr->sq = qidx;
739 }
740 sqe_hdr->total = skb->len;
741 /* Set SQE identifier which will be used later for freeing SKB */
742 sqe_hdr->sqe_id = sq->head;
743
744 /* Offload TCP/UDP checksum to HW */
745 if (skb->ip_summed == CHECKSUM_PARTIAL) {
746 sqe_hdr->ol3ptr = skb_network_offset(skb);
747 sqe_hdr->ol4ptr = skb_transport_offset(skb);
748 /* get vlan protocol Ethertype */
749 if (eth_type_vlan(skb->protocol))
750 skb->protocol = vlan_get_protocol(skb);
751
752 if (skb->protocol == htons(ETH_P_IP)) {
753 proto = ip_hdr(skb)->protocol;
754 /* In case of TSO, HW needs this to be explicitly set.
755 * So set this always, instead of adding a check.
756 */
757 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP4_CKSUM;
758 } else if (skb->protocol == htons(ETH_P_IPV6)) {
759 proto = ipv6_hdr(skb)->nexthdr;
760 sqe_hdr->ol3type = NIX_SENDL3TYPE_IP6;
761 }
762
763 if (proto == IPPROTO_TCP)
764 sqe_hdr->ol4type = NIX_SENDL4TYPE_TCP_CKSUM;
765 else if (proto == IPPROTO_UDP)
766 sqe_hdr->ol4type = NIX_SENDL4TYPE_UDP_CKSUM;
767 }
768 }
769
otx2_dma_map_tso_skb(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,int sqe,int hdr_len)770 static int otx2_dma_map_tso_skb(struct otx2_nic *pfvf,
771 struct otx2_snd_queue *sq,
772 struct sk_buff *skb, int sqe, int hdr_len)
773 {
774 int num_segs = skb_shinfo(skb)->nr_frags + 1;
775 struct sg_list *sg = &sq->sg[sqe];
776 u64 dma_addr;
777 int seg, len;
778
779 sg->num_segs = 0;
780
781 /* Get payload length at skb->data */
782 len = skb_headlen(skb) - hdr_len;
783
784 for (seg = 0; seg < num_segs; seg++) {
785 /* Skip skb->data, if there is no payload */
786 if (!seg && !len)
787 continue;
788 dma_addr = otx2_dma_map_skb_frag(pfvf, skb, seg, &len);
789 if (dma_mapping_error(pfvf->dev, dma_addr))
790 goto unmap;
791
792 /* Save DMA mapping info for later unmapping */
793 sg->dma_addr[sg->num_segs] = dma_addr;
794 sg->size[sg->num_segs] = len;
795 sg->num_segs++;
796 }
797 return 0;
798 unmap:
799 otx2_dma_unmap_skb_frags(pfvf, sg);
800 return -EINVAL;
801 }
802
otx2_tso_frag_dma_addr(struct otx2_snd_queue * sq,struct sk_buff * skb,int seg,u64 seg_addr,int hdr_len,int sqe)803 static u64 otx2_tso_frag_dma_addr(struct otx2_snd_queue *sq,
804 struct sk_buff *skb, int seg,
805 u64 seg_addr, int hdr_len, int sqe)
806 {
807 struct sg_list *sg = &sq->sg[sqe];
808 const skb_frag_t *frag;
809 int offset;
810
811 if (seg < 0)
812 return sg->dma_addr[0] + (seg_addr - (u64)skb->data);
813
814 frag = &skb_shinfo(skb)->frags[seg];
815 offset = seg_addr - (u64)skb_frag_address(frag);
816 if (skb_headlen(skb) - hdr_len)
817 seg++;
818 return sg->dma_addr[seg] + offset;
819 }
820
otx2_sqe_tso_add_sg(struct otx2_snd_queue * sq,struct sg_list * list,int * offset)821 static void otx2_sqe_tso_add_sg(struct otx2_snd_queue *sq,
822 struct sg_list *list, int *offset)
823 {
824 struct nix_sqe_sg_s *sg = NULL;
825 u16 *sg_lens = NULL;
826 u64 *iova = NULL;
827 int seg;
828
829 /* Add SG descriptors with buffer addresses */
830 for (seg = 0; seg < list->num_segs; seg++) {
831 if ((seg % MAX_SEGS_PER_SG) == 0) {
832 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
833 sg->ld_type = NIX_SEND_LDTYPE_LDD;
834 sg->subdc = NIX_SUBDC_SG;
835 sg->segs = 0;
836 sg_lens = (void *)sg;
837 iova = (void *)sg + sizeof(*sg);
838 /* Next subdc always starts at a 16byte boundary.
839 * So if sg->segs is whether 2 or 3, offset += 16bytes.
840 */
841 if ((list->num_segs - seg) >= (MAX_SEGS_PER_SG - 1))
842 *offset += sizeof(*sg) + (3 * sizeof(u64));
843 else
844 *offset += sizeof(*sg) + sizeof(u64);
845 }
846 sg_lens[frag_num(seg % MAX_SEGS_PER_SG)] = list->size[seg];
847 *iova++ = list->dma_addr[seg];
848 sg->segs++;
849 }
850 }
851
otx2_sq_append_tso(struct otx2_nic * pfvf,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)852 static void otx2_sq_append_tso(struct otx2_nic *pfvf, struct otx2_snd_queue *sq,
853 struct sk_buff *skb, u16 qidx)
854 {
855 struct netdev_queue *txq = netdev_get_tx_queue(pfvf->netdev, qidx);
856 int hdr_len, tcp_data, seg_len, pkt_len, offset;
857 struct nix_sqe_hdr_s *sqe_hdr;
858 int first_sqe = sq->head;
859 struct sg_list list;
860 struct tso_t tso;
861
862 hdr_len = tso_start(skb, &tso);
863
864 /* Map SKB's fragments to DMA.
865 * It's done here to avoid mapping for every TSO segment's packet.
866 */
867 if (otx2_dma_map_tso_skb(pfvf, sq, skb, first_sqe, hdr_len)) {
868 dev_kfree_skb_any(skb);
869 return;
870 }
871
872 netdev_tx_sent_queue(txq, skb->len);
873
874 tcp_data = skb->len - hdr_len;
875 while (tcp_data > 0) {
876 char *hdr;
877
878 seg_len = min_t(int, skb_shinfo(skb)->gso_size, tcp_data);
879 tcp_data -= seg_len;
880
881 /* Set SQE's SEND_HDR */
882 memset(sq->sqe_base, 0, sq->sqe_size);
883 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
884 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
885 offset = sizeof(*sqe_hdr);
886
887 /* Add TSO segment's pkt header */
888 hdr = sq->tso_hdrs->base + (sq->head * TSO_HEADER_SIZE);
889 tso_build_hdr(skb, hdr, &tso, seg_len, tcp_data == 0);
890 list.dma_addr[0] =
891 sq->tso_hdrs->iova + (sq->head * TSO_HEADER_SIZE);
892 list.size[0] = hdr_len;
893 list.num_segs = 1;
894
895 /* Add TSO segment's payload data fragments */
896 pkt_len = hdr_len;
897 while (seg_len > 0) {
898 int size;
899
900 size = min_t(int, tso.size, seg_len);
901
902 list.size[list.num_segs] = size;
903 list.dma_addr[list.num_segs] =
904 otx2_tso_frag_dma_addr(sq, skb,
905 tso.next_frag_idx - 1,
906 (u64)tso.data, hdr_len,
907 first_sqe);
908 list.num_segs++;
909 pkt_len += size;
910 seg_len -= size;
911 tso_build_data(skb, &tso, size);
912 }
913 sqe_hdr->total = pkt_len;
914 otx2_sqe_tso_add_sg(sq, &list, &offset);
915
916 /* DMA mappings and skb needs to be freed only after last
917 * TSO segment is transmitted out. So set 'PNC' only for
918 * last segment. Also point last segment's sqe_id to first
919 * segment's SQE index where skb address and DMA mappings
920 * are saved.
921 */
922 if (!tcp_data) {
923 sqe_hdr->pnc = 1;
924 sqe_hdr->sqe_id = first_sqe;
925 sq->sg[first_sqe].skb = (u64)skb;
926 } else {
927 sqe_hdr->pnc = 0;
928 }
929
930 sqe_hdr->sizem1 = (offset / 16) - 1;
931
932 /* Flush SQE to HW */
933 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
934 }
935 }
936
is_hw_tso_supported(struct otx2_nic * pfvf,struct sk_buff * skb)937 static bool is_hw_tso_supported(struct otx2_nic *pfvf,
938 struct sk_buff *skb)
939 {
940 int payload_len, last_seg_size;
941
942 if (test_bit(HW_TSO, &pfvf->hw.cap_flag))
943 return true;
944
945 /* On 96xx A0, HW TSO not supported */
946 if (!is_96xx_B0(pfvf->pdev))
947 return false;
948
949 /* HW has an issue due to which when the payload of the last LSO
950 * segment is shorter than 16 bytes, some header fields may not
951 * be correctly modified, hence don't offload such TSO segments.
952 */
953
954 payload_len = skb->len - skb_tcp_all_headers(skb);
955 last_seg_size = payload_len % skb_shinfo(skb)->gso_size;
956 if (last_seg_size && last_seg_size < 16)
957 return false;
958
959 return true;
960 }
961
otx2_get_sqe_count(struct otx2_nic * pfvf,struct sk_buff * skb)962 static int otx2_get_sqe_count(struct otx2_nic *pfvf, struct sk_buff *skb)
963 {
964 if (!skb_shinfo(skb)->gso_size)
965 return 1;
966
967 /* HW TSO */
968 if (is_hw_tso_supported(pfvf, skb))
969 return 1;
970
971 /* SW TSO */
972 return skb_shinfo(skb)->gso_segs;
973 }
974
otx2_validate_network_transport(struct sk_buff * skb)975 static bool otx2_validate_network_transport(struct sk_buff *skb)
976 {
977 if ((ip_hdr(skb)->protocol == IPPROTO_UDP) ||
978 (ipv6_hdr(skb)->nexthdr == IPPROTO_UDP)) {
979 struct udphdr *udph = udp_hdr(skb);
980
981 if (udph->source == htons(PTP_PORT) &&
982 udph->dest == htons(PTP_PORT))
983 return true;
984 }
985
986 return false;
987 }
988
otx2_ptp_is_sync(struct sk_buff * skb,int * offset,int * udp_csum)989 static bool otx2_ptp_is_sync(struct sk_buff *skb, int *offset, int *udp_csum)
990 {
991 struct ethhdr *eth = (struct ethhdr *)(skb->data);
992 u16 nix_offload_hlen = 0, inner_vhlen = 0;
993 u8 *data = skb->data, *msgtype;
994 __be16 proto = eth->h_proto;
995 int network_depth = 0;
996
997 /* NIX is programmed to offload outer VLAN header
998 * in case of single vlan protocol field holds Network header ETH_IP/V6
999 * in case of stacked vlan protocol field holds Inner vlan (8100)
1000 */
1001 if (skb->dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
1002 skb->dev->features & NETIF_F_HW_VLAN_STAG_TX) {
1003 if (skb->vlan_proto == htons(ETH_P_8021AD)) {
1004 /* Get vlan protocol */
1005 proto = __vlan_get_protocol(skb, eth->h_proto, NULL);
1006 /* SKB APIs like skb_transport_offset does not include
1007 * offloaded vlan header length. Need to explicitly add
1008 * the length
1009 */
1010 nix_offload_hlen = VLAN_HLEN;
1011 inner_vhlen = VLAN_HLEN;
1012 } else if (skb->vlan_proto == htons(ETH_P_8021Q)) {
1013 nix_offload_hlen = VLAN_HLEN;
1014 }
1015 } else if (eth_type_vlan(eth->h_proto)) {
1016 proto = __vlan_get_protocol(skb, eth->h_proto, &network_depth);
1017 }
1018
1019 switch (ntohs(proto)) {
1020 case ETH_P_1588:
1021 if (network_depth)
1022 *offset = network_depth;
1023 else
1024 *offset = ETH_HLEN + nix_offload_hlen +
1025 inner_vhlen;
1026 break;
1027 case ETH_P_IP:
1028 case ETH_P_IPV6:
1029 if (!otx2_validate_network_transport(skb))
1030 return false;
1031
1032 *udp_csum = 1;
1033 *offset = nix_offload_hlen + skb_transport_offset(skb) +
1034 sizeof(struct udphdr);
1035 }
1036
1037 msgtype = data + *offset;
1038
1039 /* Check PTP messageId is SYNC or not */
1040 return (*msgtype & 0xf) == 0;
1041 }
1042
otx2_set_txtstamp(struct otx2_nic * pfvf,struct sk_buff * skb,struct otx2_snd_queue * sq,int * offset)1043 static void otx2_set_txtstamp(struct otx2_nic *pfvf, struct sk_buff *skb,
1044 struct otx2_snd_queue *sq, int *offset)
1045 {
1046 struct ptpv2_tstamp *origin_tstamp;
1047 int ptp_offset = 0, udp_csum = 0;
1048 struct timespec64 ts;
1049 u64 iova;
1050
1051 if (unlikely(!skb_shinfo(skb)->gso_size &&
1052 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP))) {
1053 if (unlikely(pfvf->flags & OTX2_FLAG_PTP_ONESTEP_SYNC)) {
1054 if (otx2_ptp_is_sync(skb, &ptp_offset, &udp_csum)) {
1055 origin_tstamp = (struct ptpv2_tstamp *)
1056 ((u8 *)skb->data + ptp_offset +
1057 PTP_SYNC_SEC_OFFSET);
1058 ts = ns_to_timespec64(pfvf->ptp->tstamp);
1059 origin_tstamp->seconds_msb = htons((ts.tv_sec >> 32) & 0xffff);
1060 origin_tstamp->seconds_lsb = htonl(ts.tv_sec & 0xffffffff);
1061 origin_tstamp->nanoseconds = htonl(ts.tv_nsec);
1062 /* Point to correction field in PTP packet */
1063 ptp_offset += 8;
1064 }
1065 } else {
1066 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1067 }
1068 iova = sq->timestamps->iova + (sq->head * sizeof(u64));
1069 otx2_sqe_add_mem(sq, offset, NIX_SENDMEMALG_E_SETTSTMP, iova,
1070 ptp_offset, pfvf->ptp->base_ns, udp_csum);
1071 } else {
1072 skb_tx_timestamp(skb);
1073 }
1074 }
1075
otx2_sq_append_skb(struct net_device * netdev,struct otx2_snd_queue * sq,struct sk_buff * skb,u16 qidx)1076 bool otx2_sq_append_skb(struct net_device *netdev, struct otx2_snd_queue *sq,
1077 struct sk_buff *skb, u16 qidx)
1078 {
1079 struct netdev_queue *txq = netdev_get_tx_queue(netdev, qidx);
1080 struct otx2_nic *pfvf = netdev_priv(netdev);
1081 int offset, num_segs, free_desc;
1082 struct nix_sqe_hdr_s *sqe_hdr;
1083
1084 /* Check if there is enough room between producer
1085 * and consumer index.
1086 */
1087 free_desc = (sq->cons_head - sq->head - 1 + sq->sqe_cnt) & (sq->sqe_cnt - 1);
1088 if (free_desc < sq->sqe_thresh)
1089 return false;
1090
1091 if (free_desc < otx2_get_sqe_count(pfvf, skb))
1092 return false;
1093
1094 num_segs = skb_shinfo(skb)->nr_frags + 1;
1095
1096 /* If SKB doesn't fit in a single SQE, linearize it.
1097 * TODO: Consider adding JUMP descriptor instead.
1098 */
1099 if (unlikely(num_segs > OTX2_MAX_FRAGS_IN_SQE)) {
1100 if (__skb_linearize(skb)) {
1101 dev_kfree_skb_any(skb);
1102 return true;
1103 }
1104 num_segs = skb_shinfo(skb)->nr_frags + 1;
1105 }
1106
1107 if (skb_shinfo(skb)->gso_size && !is_hw_tso_supported(pfvf, skb)) {
1108 /* Insert vlan tag before giving pkt to tso */
1109 if (skb_vlan_tag_present(skb))
1110 skb = __vlan_hwaccel_push_inside(skb);
1111 otx2_sq_append_tso(pfvf, sq, skb, qidx);
1112 return true;
1113 }
1114
1115 /* Set SQE's SEND_HDR.
1116 * Do not clear the first 64bit as it contains constant info.
1117 */
1118 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1119 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1120 otx2_sqe_add_hdr(pfvf, sq, sqe_hdr, skb, qidx);
1121 offset = sizeof(*sqe_hdr);
1122
1123 /* Add extended header if needed */
1124 otx2_sqe_add_ext(pfvf, sq, skb, &offset);
1125
1126 /* Add SG subdesc with data frags */
1127 if (!otx2_sqe_add_sg(pfvf, sq, skb, num_segs, &offset)) {
1128 otx2_dma_unmap_skb_frags(pfvf, &sq->sg[sq->head]);
1129 return false;
1130 }
1131
1132 otx2_set_txtstamp(pfvf, skb, sq, &offset);
1133
1134 sqe_hdr->sizem1 = (offset / 16) - 1;
1135
1136 netdev_tx_sent_queue(txq, skb->len);
1137
1138 /* Flush SQE to HW */
1139 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1140
1141 return true;
1142 }
1143 EXPORT_SYMBOL(otx2_sq_append_skb);
1144
otx2_cleanup_rx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)1145 void otx2_cleanup_rx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq)
1146 {
1147 struct nix_cqe_rx_s *cqe;
1148 int processed_cqe = 0;
1149 u64 iova, pa;
1150
1151 if (pfvf->xdp_prog)
1152 xdp_rxq_info_unreg(&cq->xdp_rxq);
1153
1154 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1155 return;
1156
1157 while (cq->pend_cqe) {
1158 cqe = (struct nix_cqe_rx_s *)otx2_get_next_cqe(cq);
1159 processed_cqe++;
1160 cq->pend_cqe--;
1161
1162 if (!cqe)
1163 continue;
1164 if (cqe->sg.segs > 1) {
1165 otx2_free_rcv_seg(pfvf, cqe, cq->cq_idx);
1166 continue;
1167 }
1168 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1169 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
1170 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize, DMA_FROM_DEVICE);
1171 put_page(virt_to_page(phys_to_virt(pa)));
1172 }
1173
1174 /* Free CQEs to HW */
1175 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1176 ((u64)cq->cq_idx << 32) | processed_cqe);
1177 }
1178
otx2_cleanup_tx_cqes(struct otx2_nic * pfvf,struct otx2_cq_queue * cq)1179 void otx2_cleanup_tx_cqes(struct otx2_nic *pfvf, struct otx2_cq_queue *cq)
1180 {
1181 struct sk_buff *skb = NULL;
1182 struct otx2_snd_queue *sq;
1183 struct nix_cqe_tx_s *cqe;
1184 int processed_cqe = 0;
1185 struct sg_list *sg;
1186
1187 sq = &pfvf->qset.sq[cq->cint_idx];
1188
1189 if (otx2_nix_cq_op_status(pfvf, cq) || !cq->pend_cqe)
1190 return;
1191
1192 while (cq->pend_cqe) {
1193 cqe = (struct nix_cqe_tx_s *)otx2_get_next_cqe(cq);
1194 processed_cqe++;
1195 cq->pend_cqe--;
1196
1197 if (!cqe)
1198 continue;
1199 sg = &sq->sg[cqe->comp.sqe_id];
1200 skb = (struct sk_buff *)sg->skb;
1201 if (skb) {
1202 otx2_dma_unmap_skb_frags(pfvf, sg);
1203 dev_kfree_skb_any(skb);
1204 sg->skb = (u64)NULL;
1205 }
1206 }
1207
1208 /* Free CQEs to HW */
1209 otx2_write64(pfvf, NIX_LF_CQ_OP_DOOR,
1210 ((u64)cq->cq_idx << 32) | processed_cqe);
1211 }
1212
otx2_rxtx_enable(struct otx2_nic * pfvf,bool enable)1213 int otx2_rxtx_enable(struct otx2_nic *pfvf, bool enable)
1214 {
1215 struct msg_req *msg;
1216 int err;
1217
1218 mutex_lock(&pfvf->mbox.lock);
1219 if (enable)
1220 msg = otx2_mbox_alloc_msg_nix_lf_start_rx(&pfvf->mbox);
1221 else
1222 msg = otx2_mbox_alloc_msg_nix_lf_stop_rx(&pfvf->mbox);
1223
1224 if (!msg) {
1225 mutex_unlock(&pfvf->mbox.lock);
1226 return -ENOMEM;
1227 }
1228
1229 err = otx2_sync_mbox_msg(&pfvf->mbox);
1230 mutex_unlock(&pfvf->mbox.lock);
1231 return err;
1232 }
1233
otx2_xdp_sqe_add_sg(struct otx2_snd_queue * sq,u64 dma_addr,int len,int * offset)1234 static void otx2_xdp_sqe_add_sg(struct otx2_snd_queue *sq, u64 dma_addr,
1235 int len, int *offset)
1236 {
1237 struct nix_sqe_sg_s *sg = NULL;
1238 u64 *iova = NULL;
1239
1240 sg = (struct nix_sqe_sg_s *)(sq->sqe_base + *offset);
1241 sg->ld_type = NIX_SEND_LDTYPE_LDD;
1242 sg->subdc = NIX_SUBDC_SG;
1243 sg->segs = 1;
1244 sg->seg1_size = len;
1245 iova = (void *)sg + sizeof(*sg);
1246 *iova = dma_addr;
1247 *offset += sizeof(*sg) + sizeof(u64);
1248
1249 sq->sg[sq->head].dma_addr[0] = dma_addr;
1250 sq->sg[sq->head].size[0] = len;
1251 sq->sg[sq->head].num_segs = 1;
1252 }
1253
otx2_xdp_sq_append_pkt(struct otx2_nic * pfvf,u64 iova,int len,u16 qidx)1254 bool otx2_xdp_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, u16 qidx)
1255 {
1256 struct nix_sqe_hdr_s *sqe_hdr;
1257 struct otx2_snd_queue *sq;
1258 int offset, free_sqe;
1259
1260 sq = &pfvf->qset.sq[qidx];
1261 free_sqe = (sq->num_sqbs - *sq->aura_fc_addr) * sq->sqe_per_sqb;
1262 if (free_sqe < sq->sqe_thresh)
1263 return false;
1264
1265 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8);
1266
1267 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base);
1268
1269 if (!sqe_hdr->total) {
1270 sqe_hdr->aura = sq->aura_id;
1271 sqe_hdr->df = 1;
1272 sqe_hdr->sq = qidx;
1273 sqe_hdr->pnc = 1;
1274 }
1275 sqe_hdr->total = len;
1276 sqe_hdr->sqe_id = sq->head;
1277
1278 offset = sizeof(*sqe_hdr);
1279
1280 otx2_xdp_sqe_add_sg(sq, iova, len, &offset);
1281 sqe_hdr->sizem1 = (offset / 16) - 1;
1282 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx);
1283
1284 return true;
1285 }
1286
otx2_xdp_rcv_pkt_handler(struct otx2_nic * pfvf,struct bpf_prog * prog,struct nix_cqe_rx_s * cqe,struct otx2_cq_queue * cq)1287 static bool otx2_xdp_rcv_pkt_handler(struct otx2_nic *pfvf,
1288 struct bpf_prog *prog,
1289 struct nix_cqe_rx_s *cqe,
1290 struct otx2_cq_queue *cq)
1291 {
1292 unsigned char *hard_start, *data;
1293 int qidx = cq->cq_idx;
1294 struct xdp_buff xdp;
1295 struct page *page;
1296 u64 iova, pa;
1297 u32 act;
1298 int err;
1299
1300 iova = cqe->sg.seg_addr - OTX2_HEAD_ROOM;
1301 pa = otx2_iova_to_phys(pfvf->iommu_domain, iova);
1302 page = virt_to_page(phys_to_virt(pa));
1303
1304 xdp_init_buff(&xdp, pfvf->rbsize, &cq->xdp_rxq);
1305
1306 data = (unsigned char *)phys_to_virt(pa);
1307 hard_start = page_address(page);
1308 xdp_prepare_buff(&xdp, hard_start, data - hard_start,
1309 cqe->sg.seg_size, false);
1310
1311 act = bpf_prog_run_xdp(prog, &xdp);
1312
1313 switch (act) {
1314 case XDP_PASS:
1315 break;
1316 case XDP_TX:
1317 qidx += pfvf->hw.tx_queues;
1318 cq->pool_ptrs++;
1319 return otx2_xdp_sq_append_pkt(pfvf, iova,
1320 cqe->sg.seg_size, qidx);
1321 case XDP_REDIRECT:
1322 cq->pool_ptrs++;
1323 err = xdp_do_redirect(pfvf->netdev, &xdp, prog);
1324
1325 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1326 DMA_FROM_DEVICE);
1327 if (!err)
1328 return true;
1329 put_page(page);
1330 break;
1331 default:
1332 bpf_warn_invalid_xdp_action(pfvf->netdev, prog, act);
1333 break;
1334 case XDP_ABORTED:
1335 trace_xdp_exception(pfvf->netdev, prog, act);
1336 break;
1337 case XDP_DROP:
1338 otx2_dma_unmap_page(pfvf, iova, pfvf->rbsize,
1339 DMA_FROM_DEVICE);
1340 put_page(page);
1341 cq->pool_ptrs++;
1342 return true;
1343 }
1344 return false;
1345 }
1346