1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2
3 #include <linux/dma-mapping.h>
4 #include <linux/ip.h>
5 #include <linux/pci.h>
6 #include <linux/skbuff.h>
7 #include <linux/tcp.h>
8 #include <uapi/linux/udp.h>
9 #include "funeth.h"
10 #include "funeth_ktls.h"
11 #include "funeth_txrx.h"
12 #include "funeth_trace.h"
13 #include "fun_queue.h"
14
15 #define FUN_XDP_CLEAN_THRES 32
16 #define FUN_XDP_CLEAN_BATCH 16
17
18 /* DMA-map a packet and return the (length, DMA_address) pairs for its
19 * segments. If a mapping error occurs -ENOMEM is returned. The packet
20 * consists of an skb_shared_info and one additional address/length pair.
21 */
fun_map_pkt(struct device * dev,const struct skb_shared_info * si,void * data,unsigned int data_len,dma_addr_t * addr,unsigned int * len)22 static int fun_map_pkt(struct device *dev, const struct skb_shared_info *si,
23 void *data, unsigned int data_len,
24 dma_addr_t *addr, unsigned int *len)
25 {
26 const skb_frag_t *fp, *end;
27
28 *len = data_len;
29 *addr = dma_map_single(dev, data, *len, DMA_TO_DEVICE);
30 if (dma_mapping_error(dev, *addr))
31 return -ENOMEM;
32
33 if (!si)
34 return 0;
35
36 for (fp = si->frags, end = fp + si->nr_frags; fp < end; fp++) {
37 *++len = skb_frag_size(fp);
38 *++addr = skb_frag_dma_map(dev, fp, 0, *len, DMA_TO_DEVICE);
39 if (dma_mapping_error(dev, *addr))
40 goto unwind;
41 }
42 return 0;
43
44 unwind:
45 while (fp-- > si->frags)
46 dma_unmap_page(dev, *--addr, skb_frag_size(fp), DMA_TO_DEVICE);
47
48 dma_unmap_single(dev, addr[-1], data_len, DMA_TO_DEVICE);
49 return -ENOMEM;
50 }
51
52 /* Return the address just past the end of a Tx queue's descriptor ring.
53 * It exploits the fact that the HW writeback area is just after the end
54 * of the descriptor ring.
55 */
txq_end(const struct funeth_txq * q)56 static void *txq_end(const struct funeth_txq *q)
57 {
58 return (void *)q->hw_wb;
59 }
60
61 /* Return the amount of space within a Tx ring from the given address to the
62 * end.
63 */
txq_to_end(const struct funeth_txq * q,void * p)64 static unsigned int txq_to_end(const struct funeth_txq *q, void *p)
65 {
66 return txq_end(q) - p;
67 }
68
69 /* Return the number of Tx descriptors occupied by a Tx request. */
tx_req_ndesc(const struct fun_eth_tx_req * req)70 static unsigned int tx_req_ndesc(const struct fun_eth_tx_req *req)
71 {
72 return DIV_ROUND_UP(req->len8, FUNETH_SQE_SIZE / 8);
73 }
74
75 /* Write a gather list to the Tx descriptor at @req from @ngle address/length
76 * pairs.
77 */
fun_write_gl(const struct funeth_txq * q,struct fun_eth_tx_req * req,const dma_addr_t * addrs,const unsigned int * lens,unsigned int ngle)78 static struct fun_dataop_gl *fun_write_gl(const struct funeth_txq *q,
79 struct fun_eth_tx_req *req,
80 const dma_addr_t *addrs,
81 const unsigned int *lens,
82 unsigned int ngle)
83 {
84 struct fun_dataop_gl *gle;
85 unsigned int i;
86
87 req->len8 = (sizeof(*req) + ngle * sizeof(*gle)) / 8;
88
89 for (i = 0, gle = (struct fun_dataop_gl *)req->dataop.imm;
90 i < ngle && txq_to_end(q, gle); i++, gle++)
91 fun_dataop_gl_init(gle, 0, 0, lens[i], addrs[i]);
92
93 if (txq_to_end(q, gle) == 0) {
94 gle = (struct fun_dataop_gl *)q->desc;
95 for ( ; i < ngle; i++, gle++)
96 fun_dataop_gl_init(gle, 0, 0, lens[i], addrs[i]);
97 }
98
99 return gle;
100 }
101
tcp_hdr_doff_flags(const struct tcphdr * th)102 static __be16 tcp_hdr_doff_flags(const struct tcphdr *th)
103 {
104 return *(__be16 *)&tcp_flag_word(th);
105 }
106
fun_tls_tx(struct sk_buff * skb,struct funeth_txq * q,unsigned int * tls_len)107 static struct sk_buff *fun_tls_tx(struct sk_buff *skb, struct funeth_txq *q,
108 unsigned int *tls_len)
109 {
110 #if IS_ENABLED(CONFIG_TLS_DEVICE)
111 const struct fun_ktls_tx_ctx *tls_ctx;
112 u32 datalen, seq;
113
114 datalen = skb->len - skb_tcp_all_headers(skb);
115 if (!datalen)
116 return skb;
117
118 if (likely(!tls_offload_tx_resync_pending(skb->sk))) {
119 seq = ntohl(tcp_hdr(skb)->seq);
120 tls_ctx = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);
121
122 if (likely(tls_ctx->next_seq == seq)) {
123 *tls_len = datalen;
124 return skb;
125 }
126 if (seq - tls_ctx->next_seq < U32_MAX / 4) {
127 tls_offload_tx_resync_request(skb->sk, seq,
128 tls_ctx->next_seq);
129 }
130 }
131
132 FUN_QSTAT_INC(q, tx_tls_fallback);
133 skb = tls_encrypt_skb(skb);
134 if (!skb)
135 FUN_QSTAT_INC(q, tx_tls_drops);
136
137 return skb;
138 #else
139 return NULL;
140 #endif
141 }
142
143 /* Write as many descriptors as needed for the supplied skb starting at the
144 * current producer location. The caller has made certain enough descriptors
145 * are available.
146 *
147 * Returns the number of descriptors written, 0 on error.
148 */
write_pkt_desc(struct sk_buff * skb,struct funeth_txq * q,unsigned int tls_len)149 static unsigned int write_pkt_desc(struct sk_buff *skb, struct funeth_txq *q,
150 unsigned int tls_len)
151 {
152 unsigned int extra_bytes = 0, extra_pkts = 0;
153 unsigned int idx = q->prod_cnt & q->mask;
154 const struct skb_shared_info *shinfo;
155 unsigned int lens[MAX_SKB_FRAGS + 1];
156 dma_addr_t addrs[MAX_SKB_FRAGS + 1];
157 struct fun_eth_tx_req *req;
158 struct fun_dataop_gl *gle;
159 const struct tcphdr *th;
160 unsigned int l4_hlen;
161 unsigned int ngle;
162 u16 flags;
163
164 shinfo = skb_shinfo(skb);
165 if (unlikely(fun_map_pkt(q->dma_dev, shinfo, skb->data,
166 skb_headlen(skb), addrs, lens))) {
167 FUN_QSTAT_INC(q, tx_map_err);
168 return 0;
169 }
170
171 req = fun_tx_desc_addr(q, idx);
172 req->op = FUN_ETH_OP_TX;
173 req->len8 = 0;
174 req->flags = 0;
175 req->suboff8 = offsetof(struct fun_eth_tx_req, dataop);
176 req->repr_idn = 0;
177 req->encap_proto = 0;
178
179 if (likely(shinfo->gso_size)) {
180 if (skb->encapsulation) {
181 u16 ol4_ofst;
182
183 flags = FUN_ETH_OUTER_EN | FUN_ETH_INNER_LSO |
184 FUN_ETH_UPDATE_INNER_L4_CKSUM |
185 FUN_ETH_UPDATE_OUTER_L3_LEN;
186 if (shinfo->gso_type & (SKB_GSO_UDP_TUNNEL |
187 SKB_GSO_UDP_TUNNEL_CSUM)) {
188 flags |= FUN_ETH_UPDATE_OUTER_L4_LEN |
189 FUN_ETH_OUTER_UDP;
190 if (shinfo->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)
191 flags |= FUN_ETH_UPDATE_OUTER_L4_CKSUM;
192 ol4_ofst = skb_transport_offset(skb);
193 } else {
194 ol4_ofst = skb_inner_network_offset(skb);
195 }
196
197 if (ip_hdr(skb)->version == 4)
198 flags |= FUN_ETH_UPDATE_OUTER_L3_CKSUM;
199 else
200 flags |= FUN_ETH_OUTER_IPV6;
201
202 if (skb->inner_network_header) {
203 if (inner_ip_hdr(skb)->version == 4)
204 flags |= FUN_ETH_UPDATE_INNER_L3_CKSUM |
205 FUN_ETH_UPDATE_INNER_L3_LEN;
206 else
207 flags |= FUN_ETH_INNER_IPV6 |
208 FUN_ETH_UPDATE_INNER_L3_LEN;
209 }
210 th = inner_tcp_hdr(skb);
211 l4_hlen = __tcp_hdrlen(th);
212 fun_eth_offload_init(&req->offload, flags,
213 shinfo->gso_size,
214 tcp_hdr_doff_flags(th), 0,
215 skb_inner_network_offset(skb),
216 skb_inner_transport_offset(skb),
217 skb_network_offset(skb), ol4_ofst);
218 FUN_QSTAT_INC(q, tx_encap_tso);
219 } else if (shinfo->gso_type & SKB_GSO_UDP_L4) {
220 flags = FUN_ETH_INNER_LSO | FUN_ETH_INNER_UDP |
221 FUN_ETH_UPDATE_INNER_L4_CKSUM |
222 FUN_ETH_UPDATE_INNER_L4_LEN |
223 FUN_ETH_UPDATE_INNER_L3_LEN;
224
225 if (ip_hdr(skb)->version == 4)
226 flags |= FUN_ETH_UPDATE_INNER_L3_CKSUM;
227 else
228 flags |= FUN_ETH_INNER_IPV6;
229
230 l4_hlen = sizeof(struct udphdr);
231 fun_eth_offload_init(&req->offload, flags,
232 shinfo->gso_size,
233 cpu_to_be16(l4_hlen << 10), 0,
234 skb_network_offset(skb),
235 skb_transport_offset(skb), 0, 0);
236 FUN_QSTAT_INC(q, tx_uso);
237 } else {
238 /* HW considers one set of headers as inner */
239 flags = FUN_ETH_INNER_LSO |
240 FUN_ETH_UPDATE_INNER_L4_CKSUM |
241 FUN_ETH_UPDATE_INNER_L3_LEN;
242 if (shinfo->gso_type & SKB_GSO_TCPV6)
243 flags |= FUN_ETH_INNER_IPV6;
244 else
245 flags |= FUN_ETH_UPDATE_INNER_L3_CKSUM;
246 th = tcp_hdr(skb);
247 l4_hlen = __tcp_hdrlen(th);
248 fun_eth_offload_init(&req->offload, flags,
249 shinfo->gso_size,
250 tcp_hdr_doff_flags(th), 0,
251 skb_network_offset(skb),
252 skb_transport_offset(skb), 0, 0);
253 FUN_QSTAT_INC(q, tx_tso);
254 }
255
256 u64_stats_update_begin(&q->syncp);
257 q->stats.tx_cso += shinfo->gso_segs;
258 u64_stats_update_end(&q->syncp);
259
260 extra_pkts = shinfo->gso_segs - 1;
261 extra_bytes = (be16_to_cpu(req->offload.inner_l4_off) +
262 l4_hlen) * extra_pkts;
263 } else if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) {
264 flags = FUN_ETH_UPDATE_INNER_L4_CKSUM;
265 if (skb->csum_offset == offsetof(struct udphdr, check))
266 flags |= FUN_ETH_INNER_UDP;
267 fun_eth_offload_init(&req->offload, flags, 0, 0, 0, 0,
268 skb_checksum_start_offset(skb), 0, 0);
269 FUN_QSTAT_INC(q, tx_cso);
270 } else {
271 fun_eth_offload_init(&req->offload, 0, 0, 0, 0, 0, 0, 0, 0);
272 }
273
274 ngle = shinfo->nr_frags + 1;
275 req->dataop = FUN_DATAOP_HDR_INIT(ngle, 0, ngle, 0, skb->len);
276
277 gle = fun_write_gl(q, req, addrs, lens, ngle);
278
279 if (IS_ENABLED(CONFIG_TLS_DEVICE) && unlikely(tls_len)) {
280 struct fun_eth_tls *tls = (struct fun_eth_tls *)gle;
281 struct fun_ktls_tx_ctx *tls_ctx;
282
283 req->len8 += FUNETH_TLS_SZ / 8;
284 req->flags = cpu_to_be16(FUN_ETH_TX_TLS);
285
286 tls_ctx = tls_driver_ctx(skb->sk, TLS_OFFLOAD_CTX_DIR_TX);
287 tls->tlsid = tls_ctx->tlsid;
288 tls_ctx->next_seq += tls_len;
289
290 u64_stats_update_begin(&q->syncp);
291 q->stats.tx_tls_bytes += tls_len;
292 q->stats.tx_tls_pkts += 1 + extra_pkts;
293 u64_stats_update_end(&q->syncp);
294 }
295
296 u64_stats_update_begin(&q->syncp);
297 q->stats.tx_bytes += skb->len + extra_bytes;
298 q->stats.tx_pkts += 1 + extra_pkts;
299 u64_stats_update_end(&q->syncp);
300
301 q->info[idx].skb = skb;
302
303 trace_funeth_tx(q, skb->len, idx, req->dataop.ngather);
304 return tx_req_ndesc(req);
305 }
306
307 /* Return the number of available descriptors of a Tx queue.
308 * HW assumes head==tail means the ring is empty so we need to keep one
309 * descriptor unused.
310 */
fun_txq_avail(const struct funeth_txq * q)311 static unsigned int fun_txq_avail(const struct funeth_txq *q)
312 {
313 return q->mask - q->prod_cnt + q->cons_cnt;
314 }
315
316 /* Stop a queue if it can't handle another worst-case packet. */
fun_tx_check_stop(struct funeth_txq * q)317 static void fun_tx_check_stop(struct funeth_txq *q)
318 {
319 if (likely(fun_txq_avail(q) >= FUNETH_MAX_PKT_DESC))
320 return;
321
322 netif_tx_stop_queue(q->ndq);
323
324 /* NAPI reclaim is freeing packets in parallel with us and we may race.
325 * We have stopped the queue but check again after synchronizing with
326 * reclaim.
327 */
328 smp_mb();
329 if (likely(fun_txq_avail(q) < FUNETH_MAX_PKT_DESC))
330 FUN_QSTAT_INC(q, tx_nstops);
331 else
332 netif_tx_start_queue(q->ndq);
333 }
334
335 /* Return true if a queue has enough space to restart. Current condition is
336 * that the queue must be >= 1/4 empty.
337 */
fun_txq_may_restart(struct funeth_txq * q)338 static bool fun_txq_may_restart(struct funeth_txq *q)
339 {
340 return fun_txq_avail(q) >= q->mask / 4;
341 }
342
fun_start_xmit(struct sk_buff * skb,struct net_device * netdev)343 netdev_tx_t fun_start_xmit(struct sk_buff *skb, struct net_device *netdev)
344 {
345 struct funeth_priv *fp = netdev_priv(netdev);
346 unsigned int qid = skb_get_queue_mapping(skb);
347 struct funeth_txq *q = fp->txqs[qid];
348 unsigned int tls_len = 0;
349 unsigned int ndesc;
350
351 if (tls_is_skb_tx_device_offloaded(skb)) {
352 skb = fun_tls_tx(skb, q, &tls_len);
353 if (unlikely(!skb))
354 goto dropped;
355 }
356
357 ndesc = write_pkt_desc(skb, q, tls_len);
358 if (unlikely(!ndesc)) {
359 dev_kfree_skb_any(skb);
360 goto dropped;
361 }
362
363 q->prod_cnt += ndesc;
364 fun_tx_check_stop(q);
365
366 skb_tx_timestamp(skb);
367
368 if (__netdev_tx_sent_queue(q->ndq, skb->len, netdev_xmit_more()))
369 fun_txq_wr_db(q);
370 else
371 FUN_QSTAT_INC(q, tx_more);
372
373 return NETDEV_TX_OK;
374
375 dropped:
376 /* A dropped packet may be the last one in a xmit_more train,
377 * ring the doorbell just in case.
378 */
379 if (!netdev_xmit_more())
380 fun_txq_wr_db(q);
381 return NETDEV_TX_OK;
382 }
383
384 /* Return a Tx queue's HW head index written back to host memory. */
txq_hw_head(const struct funeth_txq * q)385 static u16 txq_hw_head(const struct funeth_txq *q)
386 {
387 return (u16)be64_to_cpu(*q->hw_wb);
388 }
389
390 /* Unmap the Tx packet starting at the given descriptor index and
391 * return the number of Tx descriptors it occupied.
392 */
fun_unmap_pkt(const struct funeth_txq * q,unsigned int idx)393 static unsigned int fun_unmap_pkt(const struct funeth_txq *q, unsigned int idx)
394 {
395 const struct fun_eth_tx_req *req = fun_tx_desc_addr(q, idx);
396 unsigned int ngle = req->dataop.ngather;
397 struct fun_dataop_gl *gle;
398
399 if (ngle) {
400 gle = (struct fun_dataop_gl *)req->dataop.imm;
401 dma_unmap_single(q->dma_dev, be64_to_cpu(gle->sgl_data),
402 be32_to_cpu(gle->sgl_len), DMA_TO_DEVICE);
403
404 for (gle++; --ngle && txq_to_end(q, gle); gle++)
405 dma_unmap_page(q->dma_dev, be64_to_cpu(gle->sgl_data),
406 be32_to_cpu(gle->sgl_len),
407 DMA_TO_DEVICE);
408
409 for (gle = (struct fun_dataop_gl *)q->desc; ngle; ngle--, gle++)
410 dma_unmap_page(q->dma_dev, be64_to_cpu(gle->sgl_data),
411 be32_to_cpu(gle->sgl_len),
412 DMA_TO_DEVICE);
413 }
414
415 return tx_req_ndesc(req);
416 }
417
418 /* Reclaim completed Tx descriptors and free their packets. Restart a stopped
419 * queue if we freed enough descriptors.
420 *
421 * Return true if we exhausted the budget while there is more work to be done.
422 */
fun_txq_reclaim(struct funeth_txq * q,int budget)423 static bool fun_txq_reclaim(struct funeth_txq *q, int budget)
424 {
425 unsigned int npkts = 0, nbytes = 0, ndesc = 0;
426 unsigned int head, limit, reclaim_idx;
427
428 /* budget may be 0, e.g., netpoll */
429 limit = budget ? budget : UINT_MAX;
430
431 for (head = txq_hw_head(q), reclaim_idx = q->cons_cnt & q->mask;
432 head != reclaim_idx && npkts < limit; head = txq_hw_head(q)) {
433 /* The HW head is continually updated, ensure we don't read
434 * descriptor state before the head tells us to reclaim it.
435 * On the enqueue side the doorbell is an implicit write
436 * barrier.
437 */
438 rmb();
439
440 do {
441 unsigned int pkt_desc = fun_unmap_pkt(q, reclaim_idx);
442 struct sk_buff *skb = q->info[reclaim_idx].skb;
443
444 trace_funeth_tx_free(q, reclaim_idx, pkt_desc, head);
445
446 nbytes += skb->len;
447 napi_consume_skb(skb, budget);
448 ndesc += pkt_desc;
449 reclaim_idx = (reclaim_idx + pkt_desc) & q->mask;
450 npkts++;
451 } while (reclaim_idx != head && npkts < limit);
452 }
453
454 q->cons_cnt += ndesc;
455 netdev_tx_completed_queue(q->ndq, npkts, nbytes);
456 smp_mb(); /* pairs with the one in fun_tx_check_stop() */
457
458 if (unlikely(netif_tx_queue_stopped(q->ndq) &&
459 fun_txq_may_restart(q))) {
460 netif_tx_wake_queue(q->ndq);
461 FUN_QSTAT_INC(q, tx_nrestarts);
462 }
463
464 return reclaim_idx != head;
465 }
466
467 /* The NAPI handler for Tx queues. */
fun_txq_napi_poll(struct napi_struct * napi,int budget)468 int fun_txq_napi_poll(struct napi_struct *napi, int budget)
469 {
470 struct fun_irq *irq = container_of(napi, struct fun_irq, napi);
471 struct funeth_txq *q = irq->txq;
472 unsigned int db_val;
473
474 if (fun_txq_reclaim(q, budget))
475 return budget; /* exhausted budget */
476
477 napi_complete(napi); /* exhausted pending work */
478 db_val = READ_ONCE(q->irq_db_val) | (q->cons_cnt & q->mask);
479 writel(db_val, q->db);
480 return 0;
481 }
482
483 /* Reclaim up to @budget completed Tx packets from a TX XDP queue. */
fun_xdpq_clean(struct funeth_txq * q,unsigned int budget)484 static unsigned int fun_xdpq_clean(struct funeth_txq *q, unsigned int budget)
485 {
486 unsigned int npkts = 0, ndesc = 0, head, reclaim_idx;
487
488 for (head = txq_hw_head(q), reclaim_idx = q->cons_cnt & q->mask;
489 head != reclaim_idx && npkts < budget; head = txq_hw_head(q)) {
490 /* The HW head is continually updated, ensure we don't read
491 * descriptor state before the head tells us to reclaim it.
492 * On the enqueue side the doorbell is an implicit write
493 * barrier.
494 */
495 rmb();
496
497 do {
498 unsigned int pkt_desc = fun_unmap_pkt(q, reclaim_idx);
499
500 xdp_return_frame(q->info[reclaim_idx].xdpf);
501
502 trace_funeth_tx_free(q, reclaim_idx, pkt_desc, head);
503
504 reclaim_idx = (reclaim_idx + pkt_desc) & q->mask;
505 ndesc += pkt_desc;
506 npkts++;
507 } while (reclaim_idx != head && npkts < budget);
508 }
509
510 q->cons_cnt += ndesc;
511 return npkts;
512 }
513
fun_xdp_tx(struct funeth_txq * q,struct xdp_frame * xdpf)514 bool fun_xdp_tx(struct funeth_txq *q, struct xdp_frame *xdpf)
515 {
516 unsigned int idx, nfrags = 1, ndesc = 1, tot_len = xdpf->len;
517 const struct skb_shared_info *si = NULL;
518 unsigned int lens[MAX_SKB_FRAGS + 1];
519 dma_addr_t dma[MAX_SKB_FRAGS + 1];
520 struct fun_eth_tx_req *req;
521
522 if (fun_txq_avail(q) < FUN_XDP_CLEAN_THRES)
523 fun_xdpq_clean(q, FUN_XDP_CLEAN_BATCH);
524
525 if (unlikely(xdp_frame_has_frags(xdpf))) {
526 si = xdp_get_shared_info_from_frame(xdpf);
527 tot_len = xdp_get_frame_len(xdpf);
528 nfrags += si->nr_frags;
529 ndesc = DIV_ROUND_UP((sizeof(*req) + nfrags *
530 sizeof(struct fun_dataop_gl)),
531 FUNETH_SQE_SIZE);
532 }
533
534 if (unlikely(fun_txq_avail(q) < ndesc)) {
535 FUN_QSTAT_INC(q, tx_xdp_full);
536 return false;
537 }
538
539 if (unlikely(fun_map_pkt(q->dma_dev, si, xdpf->data, xdpf->len, dma,
540 lens))) {
541 FUN_QSTAT_INC(q, tx_map_err);
542 return false;
543 }
544
545 idx = q->prod_cnt & q->mask;
546 req = fun_tx_desc_addr(q, idx);
547 req->op = FUN_ETH_OP_TX;
548 req->len8 = 0;
549 req->flags = 0;
550 req->suboff8 = offsetof(struct fun_eth_tx_req, dataop);
551 req->repr_idn = 0;
552 req->encap_proto = 0;
553 fun_eth_offload_init(&req->offload, 0, 0, 0, 0, 0, 0, 0, 0);
554 req->dataop = FUN_DATAOP_HDR_INIT(nfrags, 0, nfrags, 0, tot_len);
555
556 fun_write_gl(q, req, dma, lens, nfrags);
557
558 q->info[idx].xdpf = xdpf;
559
560 u64_stats_update_begin(&q->syncp);
561 q->stats.tx_bytes += tot_len;
562 q->stats.tx_pkts++;
563 u64_stats_update_end(&q->syncp);
564
565 trace_funeth_tx(q, tot_len, idx, nfrags);
566 q->prod_cnt += ndesc;
567
568 return true;
569 }
570
fun_xdp_xmit_frames(struct net_device * dev,int n,struct xdp_frame ** frames,u32 flags)571 int fun_xdp_xmit_frames(struct net_device *dev, int n,
572 struct xdp_frame **frames, u32 flags)
573 {
574 struct funeth_priv *fp = netdev_priv(dev);
575 struct funeth_txq *q, **xdpqs;
576 int i, q_idx;
577
578 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
579 return -EINVAL;
580
581 xdpqs = rcu_dereference_bh(fp->xdpqs);
582 if (unlikely(!xdpqs))
583 return -ENETDOWN;
584
585 q_idx = smp_processor_id();
586 if (unlikely(q_idx >= fp->num_xdpqs))
587 return -ENXIO;
588
589 for (q = xdpqs[q_idx], i = 0; i < n; i++)
590 if (!fun_xdp_tx(q, frames[i]))
591 break;
592
593 if (unlikely(flags & XDP_XMIT_FLUSH))
594 fun_txq_wr_db(q);
595 return i;
596 }
597
598 /* Purge a Tx queue of any queued packets. Should be called once HW access
599 * to the packets has been revoked, e.g., after the queue has been disabled.
600 */
fun_txq_purge(struct funeth_txq * q)601 static void fun_txq_purge(struct funeth_txq *q)
602 {
603 while (q->cons_cnt != q->prod_cnt) {
604 unsigned int idx = q->cons_cnt & q->mask;
605
606 q->cons_cnt += fun_unmap_pkt(q, idx);
607 dev_kfree_skb_any(q->info[idx].skb);
608 }
609 netdev_tx_reset_queue(q->ndq);
610 }
611
fun_xdpq_purge(struct funeth_txq * q)612 static void fun_xdpq_purge(struct funeth_txq *q)
613 {
614 while (q->cons_cnt != q->prod_cnt) {
615 unsigned int idx = q->cons_cnt & q->mask;
616
617 q->cons_cnt += fun_unmap_pkt(q, idx);
618 xdp_return_frame(q->info[idx].xdpf);
619 }
620 }
621
622 /* Create a Tx queue, allocating all the host resources needed. */
fun_txq_create_sw(struct net_device * dev,unsigned int qidx,unsigned int ndesc,struct fun_irq * irq)623 static struct funeth_txq *fun_txq_create_sw(struct net_device *dev,
624 unsigned int qidx,
625 unsigned int ndesc,
626 struct fun_irq *irq)
627 {
628 struct funeth_priv *fp = netdev_priv(dev);
629 struct funeth_txq *q;
630 int numa_node;
631
632 if (irq)
633 numa_node = fun_irq_node(irq); /* skb Tx queue */
634 else
635 numa_node = cpu_to_node(qidx); /* XDP Tx queue */
636
637 q = kzalloc_node(sizeof(*q), GFP_KERNEL, numa_node);
638 if (!q)
639 goto err;
640
641 q->dma_dev = &fp->pdev->dev;
642 q->desc = fun_alloc_ring_mem(q->dma_dev, ndesc, FUNETH_SQE_SIZE,
643 sizeof(*q->info), true, numa_node,
644 &q->dma_addr, (void **)&q->info,
645 &q->hw_wb);
646 if (!q->desc)
647 goto free_q;
648
649 q->netdev = dev;
650 q->mask = ndesc - 1;
651 q->qidx = qidx;
652 q->numa_node = numa_node;
653 u64_stats_init(&q->syncp);
654 q->init_state = FUN_QSTATE_INIT_SW;
655 return q;
656
657 free_q:
658 kfree(q);
659 err:
660 netdev_err(dev, "Can't allocate memory for %s queue %u\n",
661 irq ? "Tx" : "XDP", qidx);
662 return NULL;
663 }
664
fun_txq_free_sw(struct funeth_txq * q)665 static void fun_txq_free_sw(struct funeth_txq *q)
666 {
667 struct funeth_priv *fp = netdev_priv(q->netdev);
668
669 fun_free_ring_mem(q->dma_dev, q->mask + 1, FUNETH_SQE_SIZE, true,
670 q->desc, q->dma_addr, q->info);
671
672 fp->tx_packets += q->stats.tx_pkts;
673 fp->tx_bytes += q->stats.tx_bytes;
674 fp->tx_dropped += q->stats.tx_map_err;
675
676 kfree(q);
677 }
678
679 /* Allocate the device portion of a Tx queue. */
fun_txq_create_dev(struct funeth_txq * q,struct fun_irq * irq)680 int fun_txq_create_dev(struct funeth_txq *q, struct fun_irq *irq)
681 {
682 struct funeth_priv *fp = netdev_priv(q->netdev);
683 unsigned int irq_idx, ndesc = q->mask + 1;
684 int err;
685
686 q->irq = irq;
687 *q->hw_wb = 0;
688 q->prod_cnt = 0;
689 q->cons_cnt = 0;
690 irq_idx = irq ? irq->irq_idx : 0;
691
692 err = fun_sq_create(fp->fdev,
693 FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS |
694 FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR, 0,
695 FUN_HCI_ID_INVALID, ilog2(FUNETH_SQE_SIZE), ndesc,
696 q->dma_addr, fp->tx_coal_count, fp->tx_coal_usec,
697 irq_idx, 0, fp->fdev->kern_end_qid, 0,
698 &q->hw_qid, &q->db);
699 if (err)
700 goto out;
701
702 err = fun_create_and_bind_tx(fp, q->hw_qid);
703 if (err < 0)
704 goto free_devq;
705 q->ethid = err;
706
707 if (irq) {
708 irq->txq = q;
709 q->ndq = netdev_get_tx_queue(q->netdev, q->qidx);
710 q->irq_db_val = FUN_IRQ_SQ_DB(fp->tx_coal_usec,
711 fp->tx_coal_count);
712 writel(q->irq_db_val, q->db);
713 }
714
715 q->init_state = FUN_QSTATE_INIT_FULL;
716 netif_info(fp, ifup, q->netdev,
717 "%s queue %u, depth %u, HW qid %u, IRQ idx %u, eth id %u, node %d\n",
718 irq ? "Tx" : "XDP", q->qidx, ndesc, q->hw_qid, irq_idx,
719 q->ethid, q->numa_node);
720 return 0;
721
722 free_devq:
723 fun_destroy_sq(fp->fdev, q->hw_qid);
724 out:
725 netdev_err(q->netdev,
726 "Failed to create %s queue %u on device, error %d\n",
727 irq ? "Tx" : "XDP", q->qidx, err);
728 return err;
729 }
730
fun_txq_free_dev(struct funeth_txq * q)731 static void fun_txq_free_dev(struct funeth_txq *q)
732 {
733 struct funeth_priv *fp = netdev_priv(q->netdev);
734
735 if (q->init_state < FUN_QSTATE_INIT_FULL)
736 return;
737
738 netif_info(fp, ifdown, q->netdev,
739 "Freeing %s queue %u (id %u), IRQ %u, ethid %u\n",
740 q->irq ? "Tx" : "XDP", q->qidx, q->hw_qid,
741 q->irq ? q->irq->irq_idx : 0, q->ethid);
742
743 fun_destroy_sq(fp->fdev, q->hw_qid);
744 fun_res_destroy(fp->fdev, FUN_ADMIN_OP_ETH, 0, q->ethid);
745
746 if (q->irq) {
747 q->irq->txq = NULL;
748 fun_txq_purge(q);
749 } else {
750 fun_xdpq_purge(q);
751 }
752
753 q->init_state = FUN_QSTATE_INIT_SW;
754 }
755
756 /* Create or advance a Tx queue, allocating all the host and device resources
757 * needed to reach the target state.
758 */
funeth_txq_create(struct net_device * dev,unsigned int qidx,unsigned int ndesc,struct fun_irq * irq,int state,struct funeth_txq ** qp)759 int funeth_txq_create(struct net_device *dev, unsigned int qidx,
760 unsigned int ndesc, struct fun_irq *irq, int state,
761 struct funeth_txq **qp)
762 {
763 struct funeth_txq *q = *qp;
764 int err;
765
766 if (!q)
767 q = fun_txq_create_sw(dev, qidx, ndesc, irq);
768 if (!q)
769 return -ENOMEM;
770
771 if (q->init_state >= state)
772 goto out;
773
774 err = fun_txq_create_dev(q, irq);
775 if (err) {
776 if (!*qp)
777 fun_txq_free_sw(q);
778 return err;
779 }
780
781 out:
782 *qp = q;
783 return 0;
784 }
785
786 /* Free Tx queue resources until it reaches the target state.
787 * The queue must be already disconnected from the stack.
788 */
funeth_txq_free(struct funeth_txq * q,int state)789 struct funeth_txq *funeth_txq_free(struct funeth_txq *q, int state)
790 {
791 if (state < FUN_QSTATE_INIT_FULL)
792 fun_txq_free_dev(q);
793
794 if (state == FUN_QSTATE_DESTROYED) {
795 fun_txq_free_sw(q);
796 q = NULL;
797 }
798
799 return q;
800 }
801