1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3
4 #include <uapi/linux/bpf.h>
5
6 #include <linux/inetdevice.h>
7 #include <linux/etherdevice.h>
8 #include <linux/ethtool.h>
9 #include <linux/filter.h>
10 #include <linux/mm.h>
11
12 #include <net/checksum.h>
13 #include <net/ip6_checksum.h>
14
15 #include "mana.h"
16
17 /* Microsoft Azure Network Adapter (MANA) functions */
18
mana_open(struct net_device * ndev)19 static int mana_open(struct net_device *ndev)
20 {
21 struct mana_port_context *apc = netdev_priv(ndev);
22 int err;
23
24 err = mana_alloc_queues(ndev);
25 if (err)
26 return err;
27
28 apc->port_is_up = true;
29
30 /* Ensure port state updated before txq state */
31 smp_wmb();
32
33 netif_carrier_on(ndev);
34 netif_tx_wake_all_queues(ndev);
35
36 return 0;
37 }
38
mana_close(struct net_device * ndev)39 static int mana_close(struct net_device *ndev)
40 {
41 struct mana_port_context *apc = netdev_priv(ndev);
42
43 if (!apc->port_is_up)
44 return 0;
45
46 return mana_detach(ndev, true);
47 }
48
mana_can_tx(struct gdma_queue * wq)49 static bool mana_can_tx(struct gdma_queue *wq)
50 {
51 return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;
52 }
53
mana_checksum_info(struct sk_buff * skb)54 static unsigned int mana_checksum_info(struct sk_buff *skb)
55 {
56 if (skb->protocol == htons(ETH_P_IP)) {
57 struct iphdr *ip = ip_hdr(skb);
58
59 if (ip->protocol == IPPROTO_TCP)
60 return IPPROTO_TCP;
61
62 if (ip->protocol == IPPROTO_UDP)
63 return IPPROTO_UDP;
64 } else if (skb->protocol == htons(ETH_P_IPV6)) {
65 struct ipv6hdr *ip6 = ipv6_hdr(skb);
66
67 if (ip6->nexthdr == IPPROTO_TCP)
68 return IPPROTO_TCP;
69
70 if (ip6->nexthdr == IPPROTO_UDP)
71 return IPPROTO_UDP;
72 }
73
74 /* No csum offloading */
75 return 0;
76 }
77
mana_map_skb(struct sk_buff * skb,struct mana_port_context * apc,struct mana_tx_package * tp)78 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc,
79 struct mana_tx_package *tp)
80 {
81 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
82 struct gdma_dev *gd = apc->ac->gdma_dev;
83 struct gdma_context *gc;
84 struct device *dev;
85 skb_frag_t *frag;
86 dma_addr_t da;
87 int i;
88
89 gc = gd->gdma_context;
90 dev = gc->dev;
91 da = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
92
93 if (dma_mapping_error(dev, da))
94 return -ENOMEM;
95
96 ash->dma_handle[0] = da;
97 ash->size[0] = skb_headlen(skb);
98
99 tp->wqe_req.sgl[0].address = ash->dma_handle[0];
100 tp->wqe_req.sgl[0].mem_key = gd->gpa_mkey;
101 tp->wqe_req.sgl[0].size = ash->size[0];
102
103 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
104 frag = &skb_shinfo(skb)->frags[i];
105 da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
106 DMA_TO_DEVICE);
107
108 if (dma_mapping_error(dev, da))
109 goto frag_err;
110
111 ash->dma_handle[i + 1] = da;
112 ash->size[i + 1] = skb_frag_size(frag);
113
114 tp->wqe_req.sgl[i + 1].address = ash->dma_handle[i + 1];
115 tp->wqe_req.sgl[i + 1].mem_key = gd->gpa_mkey;
116 tp->wqe_req.sgl[i + 1].size = ash->size[i + 1];
117 }
118
119 return 0;
120
121 frag_err:
122 for (i = i - 1; i >= 0; i--)
123 dma_unmap_page(dev, ash->dma_handle[i + 1], ash->size[i + 1],
124 DMA_TO_DEVICE);
125
126 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE);
127
128 return -ENOMEM;
129 }
130
mana_start_xmit(struct sk_buff * skb,struct net_device * ndev)131 int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
132 {
133 enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT;
134 struct mana_port_context *apc = netdev_priv(ndev);
135 u16 txq_idx = skb_get_queue_mapping(skb);
136 struct gdma_dev *gd = apc->ac->gdma_dev;
137 bool ipv4 = false, ipv6 = false;
138 struct mana_tx_package pkg = {};
139 struct netdev_queue *net_txq;
140 struct mana_stats_tx *tx_stats;
141 struct gdma_queue *gdma_sq;
142 unsigned int csum_type;
143 struct mana_txq *txq;
144 struct mana_cq *cq;
145 int err, len;
146
147 if (unlikely(!apc->port_is_up))
148 goto tx_drop;
149
150 if (skb_cow_head(skb, MANA_HEADROOM))
151 goto tx_drop_count;
152
153 txq = &apc->tx_qp[txq_idx].txq;
154 gdma_sq = txq->gdma_sq;
155 cq = &apc->tx_qp[txq_idx].tx_cq;
156
157 pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
158 pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
159
160 if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {
161 pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;
162 pkt_fmt = MANA_LONG_PKT_FMT;
163 } else {
164 pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;
165 }
166
167 pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;
168
169 if (pkt_fmt == MANA_SHORT_PKT_FMT)
170 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);
171 else
172 pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);
173
174 pkg.wqe_req.inline_oob_data = &pkg.tx_oob;
175 pkg.wqe_req.flags = 0;
176 pkg.wqe_req.client_data_unit = 0;
177
178 pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags;
179 WARN_ON_ONCE(pkg.wqe_req.num_sge > 30);
180
181 if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) {
182 pkg.wqe_req.sgl = pkg.sgl_array;
183 } else {
184 pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge,
185 sizeof(struct gdma_sge),
186 GFP_ATOMIC);
187 if (!pkg.sgl_ptr)
188 goto tx_drop_count;
189
190 pkg.wqe_req.sgl = pkg.sgl_ptr;
191 }
192
193 if (skb->protocol == htons(ETH_P_IP))
194 ipv4 = true;
195 else if (skb->protocol == htons(ETH_P_IPV6))
196 ipv6 = true;
197
198 if (skb_is_gso(skb)) {
199 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
200 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
201
202 pkg.tx_oob.s_oob.comp_iphdr_csum = 1;
203 pkg.tx_oob.s_oob.comp_tcp_csum = 1;
204 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
205
206 pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size;
207 pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0;
208 if (ipv4) {
209 ip_hdr(skb)->tot_len = 0;
210 ip_hdr(skb)->check = 0;
211 tcp_hdr(skb)->check =
212 ~csum_tcpudp_magic(ip_hdr(skb)->saddr,
213 ip_hdr(skb)->daddr, 0,
214 IPPROTO_TCP, 0);
215 } else {
216 ipv6_hdr(skb)->payload_len = 0;
217 tcp_hdr(skb)->check =
218 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
219 &ipv6_hdr(skb)->daddr, 0,
220 IPPROTO_TCP, 0);
221 }
222 } else if (skb->ip_summed == CHECKSUM_PARTIAL) {
223 csum_type = mana_checksum_info(skb);
224
225 if (csum_type == IPPROTO_TCP) {
226 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
227 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
228
229 pkg.tx_oob.s_oob.comp_tcp_csum = 1;
230 pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
231
232 } else if (csum_type == IPPROTO_UDP) {
233 pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
234 pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
235
236 pkg.tx_oob.s_oob.comp_udp_csum = 1;
237 } else {
238 /* Can't do offload of this type of checksum */
239 if (skb_checksum_help(skb))
240 goto free_sgl_ptr;
241 }
242 }
243
244 if (mana_map_skb(skb, apc, &pkg))
245 goto free_sgl_ptr;
246
247 skb_queue_tail(&txq->pending_skbs, skb);
248
249 len = skb->len;
250 net_txq = netdev_get_tx_queue(ndev, txq_idx);
251
252 err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req,
253 (struct gdma_posted_wqe_info *)skb->cb);
254 if (!mana_can_tx(gdma_sq)) {
255 netif_tx_stop_queue(net_txq);
256 apc->eth_stats.stop_queue++;
257 }
258
259 if (err) {
260 (void)skb_dequeue_tail(&txq->pending_skbs);
261 netdev_warn(ndev, "Failed to post TX OOB: %d\n", err);
262 err = NETDEV_TX_BUSY;
263 goto tx_busy;
264 }
265
266 err = NETDEV_TX_OK;
267 atomic_inc(&txq->pending_sends);
268
269 mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq);
270
271 /* skb may be freed after mana_gd_post_work_request. Do not use it. */
272 skb = NULL;
273
274 tx_stats = &txq->stats;
275 u64_stats_update_begin(&tx_stats->syncp);
276 tx_stats->packets++;
277 tx_stats->bytes += len;
278 u64_stats_update_end(&tx_stats->syncp);
279
280 tx_busy:
281 if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) {
282 netif_tx_wake_queue(net_txq);
283 apc->eth_stats.wake_queue++;
284 }
285
286 kfree(pkg.sgl_ptr);
287 return err;
288
289 free_sgl_ptr:
290 kfree(pkg.sgl_ptr);
291 tx_drop_count:
292 ndev->stats.tx_dropped++;
293 tx_drop:
294 dev_kfree_skb_any(skb);
295 return NETDEV_TX_OK;
296 }
297
mana_get_stats64(struct net_device * ndev,struct rtnl_link_stats64 * st)298 static void mana_get_stats64(struct net_device *ndev,
299 struct rtnl_link_stats64 *st)
300 {
301 struct mana_port_context *apc = netdev_priv(ndev);
302 unsigned int num_queues = apc->num_queues;
303 struct mana_stats_rx *rx_stats;
304 struct mana_stats_tx *tx_stats;
305 unsigned int start;
306 u64 packets, bytes;
307 int q;
308
309 if (!apc->port_is_up)
310 return;
311
312 netdev_stats_to_stats64(st, &ndev->stats);
313
314 for (q = 0; q < num_queues; q++) {
315 rx_stats = &apc->rxqs[q]->stats;
316
317 do {
318 start = u64_stats_fetch_begin_irq(&rx_stats->syncp);
319 packets = rx_stats->packets;
320 bytes = rx_stats->bytes;
321 } while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start));
322
323 st->rx_packets += packets;
324 st->rx_bytes += bytes;
325 }
326
327 for (q = 0; q < num_queues; q++) {
328 tx_stats = &apc->tx_qp[q].txq.stats;
329
330 do {
331 start = u64_stats_fetch_begin_irq(&tx_stats->syncp);
332 packets = tx_stats->packets;
333 bytes = tx_stats->bytes;
334 } while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start));
335
336 st->tx_packets += packets;
337 st->tx_bytes += bytes;
338 }
339 }
340
mana_get_tx_queue(struct net_device * ndev,struct sk_buff * skb,int old_q)341 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb,
342 int old_q)
343 {
344 struct mana_port_context *apc = netdev_priv(ndev);
345 u32 hash = skb_get_hash(skb);
346 struct sock *sk = skb->sk;
347 int txq;
348
349 txq = apc->indir_table[hash & MANA_INDIRECT_TABLE_MASK];
350
351 if (txq != old_q && sk && sk_fullsock(sk) &&
352 rcu_access_pointer(sk->sk_dst_cache))
353 sk_tx_queue_set(sk, txq);
354
355 return txq;
356 }
357
mana_select_queue(struct net_device * ndev,struct sk_buff * skb,struct net_device * sb_dev)358 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,
359 struct net_device *sb_dev)
360 {
361 int txq;
362
363 if (ndev->real_num_tx_queues == 1)
364 return 0;
365
366 txq = sk_tx_queue_get(skb->sk);
367
368 if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) {
369 if (skb_rx_queue_recorded(skb))
370 txq = skb_get_rx_queue(skb);
371 else
372 txq = mana_get_tx_queue(ndev, skb, txq);
373 }
374
375 return txq;
376 }
377
378 static const struct net_device_ops mana_devops = {
379 .ndo_open = mana_open,
380 .ndo_stop = mana_close,
381 .ndo_select_queue = mana_select_queue,
382 .ndo_start_xmit = mana_start_xmit,
383 .ndo_validate_addr = eth_validate_addr,
384 .ndo_get_stats64 = mana_get_stats64,
385 .ndo_bpf = mana_bpf,
386 .ndo_xdp_xmit = mana_xdp_xmit,
387 };
388
mana_cleanup_port_context(struct mana_port_context * apc)389 static void mana_cleanup_port_context(struct mana_port_context *apc)
390 {
391 kfree(apc->rxqs);
392 apc->rxqs = NULL;
393 }
394
mana_init_port_context(struct mana_port_context * apc)395 static int mana_init_port_context(struct mana_port_context *apc)
396 {
397 apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *),
398 GFP_KERNEL);
399
400 return !apc->rxqs ? -ENOMEM : 0;
401 }
402
mana_send_request(struct mana_context * ac,void * in_buf,u32 in_len,void * out_buf,u32 out_len)403 static int mana_send_request(struct mana_context *ac, void *in_buf,
404 u32 in_len, void *out_buf, u32 out_len)
405 {
406 struct gdma_context *gc = ac->gdma_dev->gdma_context;
407 struct gdma_resp_hdr *resp = out_buf;
408 struct gdma_req_hdr *req = in_buf;
409 struct device *dev = gc->dev;
410 static atomic_t activity_id;
411 int err;
412
413 req->dev_id = gc->mana.dev_id;
414 req->activity_id = atomic_inc_return(&activity_id);
415
416 err = mana_gd_send_request(gc, in_len, in_buf, out_len,
417 out_buf);
418 if (err || resp->status) {
419 dev_err(dev, "Failed to send mana message: %d, 0x%x\n",
420 err, resp->status);
421 return err ? err : -EPROTO;
422 }
423
424 if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 ||
425 req->activity_id != resp->activity_id) {
426 dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n",
427 req->dev_id.as_uint32, resp->dev_id.as_uint32,
428 req->activity_id, resp->activity_id);
429 return -EPROTO;
430 }
431
432 return 0;
433 }
434
mana_verify_resp_hdr(const struct gdma_resp_hdr * resp_hdr,const enum mana_command_code expected_code,const u32 min_size)435 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr,
436 const enum mana_command_code expected_code,
437 const u32 min_size)
438 {
439 if (resp_hdr->response.msg_type != expected_code)
440 return -EPROTO;
441
442 if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)
443 return -EPROTO;
444
445 if (resp_hdr->response.msg_size < min_size)
446 return -EPROTO;
447
448 return 0;
449 }
450
mana_pf_register_hw_vport(struct mana_port_context * apc)451 static int mana_pf_register_hw_vport(struct mana_port_context *apc)
452 {
453 struct mana_register_hw_vport_resp resp = {};
454 struct mana_register_hw_vport_req req = {};
455 int err;
456
457 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT,
458 sizeof(req), sizeof(resp));
459 req.attached_gfid = 1;
460 req.is_pf_default_vport = 1;
461 req.allow_all_ether_types = 1;
462
463 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
464 sizeof(resp));
465 if (err) {
466 netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err);
467 return err;
468 }
469
470 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT,
471 sizeof(resp));
472 if (err || resp.hdr.status) {
473 netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n",
474 err, resp.hdr.status);
475 return err ? err : -EPROTO;
476 }
477
478 apc->port_handle = resp.hw_vport_handle;
479 return 0;
480 }
481
mana_pf_deregister_hw_vport(struct mana_port_context * apc)482 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc)
483 {
484 struct mana_deregister_hw_vport_resp resp = {};
485 struct mana_deregister_hw_vport_req req = {};
486 int err;
487
488 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT,
489 sizeof(req), sizeof(resp));
490 req.hw_vport_handle = apc->port_handle;
491
492 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
493 sizeof(resp));
494 if (err) {
495 netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n",
496 err);
497 return;
498 }
499
500 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT,
501 sizeof(resp));
502 if (err || resp.hdr.status)
503 netdev_err(apc->ndev,
504 "Failed to deregister hw vPort: %d, 0x%x\n",
505 err, resp.hdr.status);
506 }
507
mana_pf_register_filter(struct mana_port_context * apc)508 static int mana_pf_register_filter(struct mana_port_context *apc)
509 {
510 struct mana_register_filter_resp resp = {};
511 struct mana_register_filter_req req = {};
512 int err;
513
514 mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER,
515 sizeof(req), sizeof(resp));
516 req.vport = apc->port_handle;
517 memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN);
518
519 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
520 sizeof(resp));
521 if (err) {
522 netdev_err(apc->ndev, "Failed to register filter: %d\n", err);
523 return err;
524 }
525
526 err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER,
527 sizeof(resp));
528 if (err || resp.hdr.status) {
529 netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n",
530 err, resp.hdr.status);
531 return err ? err : -EPROTO;
532 }
533
534 apc->pf_filter_handle = resp.filter_handle;
535 return 0;
536 }
537
mana_pf_deregister_filter(struct mana_port_context * apc)538 static void mana_pf_deregister_filter(struct mana_port_context *apc)
539 {
540 struct mana_deregister_filter_resp resp = {};
541 struct mana_deregister_filter_req req = {};
542 int err;
543
544 mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER,
545 sizeof(req), sizeof(resp));
546 req.filter_handle = apc->pf_filter_handle;
547
548 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
549 sizeof(resp));
550 if (err) {
551 netdev_err(apc->ndev, "Failed to unregister filter: %d\n",
552 err);
553 return;
554 }
555
556 err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER,
557 sizeof(resp));
558 if (err || resp.hdr.status)
559 netdev_err(apc->ndev,
560 "Failed to deregister filter: %d, 0x%x\n",
561 err, resp.hdr.status);
562 }
563
mana_query_device_cfg(struct mana_context * ac,u32 proto_major_ver,u32 proto_minor_ver,u32 proto_micro_ver,u16 * max_num_vports)564 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
565 u32 proto_minor_ver, u32 proto_micro_ver,
566 u16 *max_num_vports)
567 {
568 struct gdma_context *gc = ac->gdma_dev->gdma_context;
569 struct mana_query_device_cfg_resp resp = {};
570 struct mana_query_device_cfg_req req = {};
571 struct device *dev = gc->dev;
572 int err = 0;
573
574 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG,
575 sizeof(req), sizeof(resp));
576 req.proto_major_ver = proto_major_ver;
577 req.proto_minor_ver = proto_minor_ver;
578 req.proto_micro_ver = proto_micro_ver;
579
580 err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp));
581 if (err) {
582 dev_err(dev, "Failed to query config: %d", err);
583 return err;
584 }
585
586 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG,
587 sizeof(resp));
588 if (err || resp.hdr.status) {
589 dev_err(dev, "Invalid query result: %d, 0x%x\n", err,
590 resp.hdr.status);
591 if (!err)
592 err = -EPROTO;
593 return err;
594 }
595
596 *max_num_vports = resp.max_num_vports;
597
598 return 0;
599 }
600
mana_query_vport_cfg(struct mana_port_context * apc,u32 vport_index,u32 * max_sq,u32 * max_rq,u32 * num_indir_entry)601 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,
602 u32 *max_sq, u32 *max_rq, u32 *num_indir_entry)
603 {
604 struct mana_query_vport_cfg_resp resp = {};
605 struct mana_query_vport_cfg_req req = {};
606 int err;
607
608 mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG,
609 sizeof(req), sizeof(resp));
610
611 req.vport_index = vport_index;
612
613 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
614 sizeof(resp));
615 if (err)
616 return err;
617
618 err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG,
619 sizeof(resp));
620 if (err)
621 return err;
622
623 if (resp.hdr.status)
624 return -EPROTO;
625
626 *max_sq = resp.max_num_sq;
627 *max_rq = resp.max_num_rq;
628 *num_indir_entry = resp.num_indirection_ent;
629
630 apc->port_handle = resp.vport;
631 ether_addr_copy(apc->mac_addr, resp.mac_addr);
632
633 return 0;
634 }
635
mana_cfg_vport(struct mana_port_context * apc,u32 protection_dom_id,u32 doorbell_pg_id)636 static int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
637 u32 doorbell_pg_id)
638 {
639 struct mana_config_vport_resp resp = {};
640 struct mana_config_vport_req req = {};
641 int err;
642
643 mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX,
644 sizeof(req), sizeof(resp));
645 req.vport = apc->port_handle;
646 req.pdid = protection_dom_id;
647 req.doorbell_pageid = doorbell_pg_id;
648
649 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
650 sizeof(resp));
651 if (err) {
652 netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err);
653 goto out;
654 }
655
656 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX,
657 sizeof(resp));
658 if (err || resp.hdr.status) {
659 netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n",
660 err, resp.hdr.status);
661 if (!err)
662 err = -EPROTO;
663
664 goto out;
665 }
666
667 apc->tx_shortform_allowed = resp.short_form_allowed;
668 apc->tx_vp_offset = resp.tx_vport_offset;
669 out:
670 return err;
671 }
672
mana_cfg_vport_steering(struct mana_port_context * apc,enum TRI_STATE rx,bool update_default_rxobj,bool update_key,bool update_tab)673 static int mana_cfg_vport_steering(struct mana_port_context *apc,
674 enum TRI_STATE rx,
675 bool update_default_rxobj, bool update_key,
676 bool update_tab)
677 {
678 u16 num_entries = MANA_INDIRECT_TABLE_SIZE;
679 struct mana_cfg_rx_steer_req *req = NULL;
680 struct mana_cfg_rx_steer_resp resp = {};
681 struct net_device *ndev = apc->ndev;
682 mana_handle_t *req_indir_tab;
683 u32 req_buf_size;
684 int err;
685
686 req_buf_size = sizeof(*req) + sizeof(mana_handle_t) * num_entries;
687 req = kzalloc(req_buf_size, GFP_KERNEL);
688 if (!req)
689 return -ENOMEM;
690
691 mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
692 sizeof(resp));
693
694 req->vport = apc->port_handle;
695 req->num_indir_entries = num_entries;
696 req->indir_tab_offset = sizeof(*req);
697 req->rx_enable = rx;
698 req->rss_enable = apc->rss_state;
699 req->update_default_rxobj = update_default_rxobj;
700 req->update_hashkey = update_key;
701 req->update_indir_tab = update_tab;
702 req->default_rxobj = apc->default_rxobj;
703
704 if (update_key)
705 memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
706
707 if (update_tab) {
708 req_indir_tab = (mana_handle_t *)(req + 1);
709 memcpy(req_indir_tab, apc->rxobj_table,
710 req->num_indir_entries * sizeof(mana_handle_t));
711 }
712
713 err = mana_send_request(apc->ac, req, req_buf_size, &resp,
714 sizeof(resp));
715 if (err) {
716 netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
717 goto out;
718 }
719
720 err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX,
721 sizeof(resp));
722 if (err) {
723 netdev_err(ndev, "vPort RX configuration failed: %d\n", err);
724 goto out;
725 }
726
727 if (resp.hdr.status) {
728 netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
729 resp.hdr.status);
730 err = -EPROTO;
731 }
732 out:
733 kfree(req);
734 return err;
735 }
736
mana_create_wq_obj(struct mana_port_context * apc,mana_handle_t vport,u32 wq_type,struct mana_obj_spec * wq_spec,struct mana_obj_spec * cq_spec,mana_handle_t * wq_obj)737 static int mana_create_wq_obj(struct mana_port_context *apc,
738 mana_handle_t vport,
739 u32 wq_type, struct mana_obj_spec *wq_spec,
740 struct mana_obj_spec *cq_spec,
741 mana_handle_t *wq_obj)
742 {
743 struct mana_create_wqobj_resp resp = {};
744 struct mana_create_wqobj_req req = {};
745 struct net_device *ndev = apc->ndev;
746 int err;
747
748 mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
749 sizeof(req), sizeof(resp));
750 req.vport = vport;
751 req.wq_type = wq_type;
752 req.wq_gdma_region = wq_spec->gdma_region;
753 req.cq_gdma_region = cq_spec->gdma_region;
754 req.wq_size = wq_spec->queue_size;
755 req.cq_size = cq_spec->queue_size;
756 req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
757 req.cq_parent_qid = cq_spec->attached_eq;
758
759 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
760 sizeof(resp));
761 if (err) {
762 netdev_err(ndev, "Failed to create WQ object: %d\n", err);
763 goto out;
764 }
765
766 err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ,
767 sizeof(resp));
768 if (err || resp.hdr.status) {
769 netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err,
770 resp.hdr.status);
771 if (!err)
772 err = -EPROTO;
773 goto out;
774 }
775
776 if (resp.wq_obj == INVALID_MANA_HANDLE) {
777 netdev_err(ndev, "Got an invalid WQ object handle\n");
778 err = -EPROTO;
779 goto out;
780 }
781
782 *wq_obj = resp.wq_obj;
783 wq_spec->queue_index = resp.wq_id;
784 cq_spec->queue_index = resp.cq_id;
785
786 return 0;
787 out:
788 return err;
789 }
790
mana_destroy_wq_obj(struct mana_port_context * apc,u32 wq_type,mana_handle_t wq_obj)791 static void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
792 mana_handle_t wq_obj)
793 {
794 struct mana_destroy_wqobj_resp resp = {};
795 struct mana_destroy_wqobj_req req = {};
796 struct net_device *ndev = apc->ndev;
797 int err;
798
799 mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ,
800 sizeof(req), sizeof(resp));
801 req.wq_type = wq_type;
802 req.wq_obj_handle = wq_obj;
803
804 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
805 sizeof(resp));
806 if (err) {
807 netdev_err(ndev, "Failed to destroy WQ object: %d\n", err);
808 return;
809 }
810
811 err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ,
812 sizeof(resp));
813 if (err || resp.hdr.status)
814 netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err,
815 resp.hdr.status);
816 }
817
mana_destroy_eq(struct mana_context * ac)818 static void mana_destroy_eq(struct mana_context *ac)
819 {
820 struct gdma_context *gc = ac->gdma_dev->gdma_context;
821 struct gdma_queue *eq;
822 int i;
823
824 if (!ac->eqs)
825 return;
826
827 for (i = 0; i < gc->max_num_queues; i++) {
828 eq = ac->eqs[i].eq;
829 if (!eq)
830 continue;
831
832 mana_gd_destroy_queue(gc, eq);
833 }
834
835 kfree(ac->eqs);
836 ac->eqs = NULL;
837 }
838
mana_create_eq(struct mana_context * ac)839 static int mana_create_eq(struct mana_context *ac)
840 {
841 struct gdma_dev *gd = ac->gdma_dev;
842 struct gdma_context *gc = gd->gdma_context;
843 struct gdma_queue_spec spec = {};
844 int err;
845 int i;
846
847 ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq),
848 GFP_KERNEL);
849 if (!ac->eqs)
850 return -ENOMEM;
851
852 spec.type = GDMA_EQ;
853 spec.monitor_avl_buf = false;
854 spec.queue_size = EQ_SIZE;
855 spec.eq.callback = NULL;
856 spec.eq.context = ac->eqs;
857 spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
858
859 for (i = 0; i < gc->max_num_queues; i++) {
860 err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);
861 if (err)
862 goto out;
863 }
864
865 return 0;
866 out:
867 mana_destroy_eq(ac);
868 return err;
869 }
870
mana_fence_rq(struct mana_port_context * apc,struct mana_rxq * rxq)871 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq)
872 {
873 struct mana_fence_rq_resp resp = {};
874 struct mana_fence_rq_req req = {};
875 int err;
876
877 init_completion(&rxq->fence_event);
878
879 mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ,
880 sizeof(req), sizeof(resp));
881 req.wq_obj_handle = rxq->rxobj;
882
883 err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
884 sizeof(resp));
885 if (err) {
886 netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n",
887 rxq->rxq_idx, err);
888 return err;
889 }
890
891 err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp));
892 if (err || resp.hdr.status) {
893 netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n",
894 rxq->rxq_idx, err, resp.hdr.status);
895 if (!err)
896 err = -EPROTO;
897
898 return err;
899 }
900
901 if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) {
902 netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n",
903 rxq->rxq_idx);
904 return -ETIMEDOUT;
905 }
906
907 return 0;
908 }
909
mana_fence_rqs(struct mana_port_context * apc)910 static void mana_fence_rqs(struct mana_port_context *apc)
911 {
912 unsigned int rxq_idx;
913 struct mana_rxq *rxq;
914 int err;
915
916 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
917 rxq = apc->rxqs[rxq_idx];
918 err = mana_fence_rq(apc, rxq);
919
920 /* In case of any error, use sleep instead. */
921 if (err)
922 msleep(100);
923 }
924 }
925
mana_move_wq_tail(struct gdma_queue * wq,u32 num_units)926 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units)
927 {
928 u32 used_space_old;
929 u32 used_space_new;
930
931 used_space_old = wq->head - wq->tail;
932 used_space_new = wq->head - (wq->tail + num_units);
933
934 if (WARN_ON_ONCE(used_space_new > used_space_old))
935 return -ERANGE;
936
937 wq->tail += num_units;
938 return 0;
939 }
940
mana_unmap_skb(struct sk_buff * skb,struct mana_port_context * apc)941 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc)
942 {
943 struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
944 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
945 struct device *dev = gc->dev;
946 int i;
947
948 dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE);
949
950 for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++)
951 dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
952 DMA_TO_DEVICE);
953 }
954
mana_poll_tx_cq(struct mana_cq * cq)955 static void mana_poll_tx_cq(struct mana_cq *cq)
956 {
957 struct gdma_comp *completions = cq->gdma_comp_buf;
958 struct gdma_posted_wqe_info *wqe_info;
959 unsigned int pkt_transmitted = 0;
960 unsigned int wqe_unit_cnt = 0;
961 struct mana_txq *txq = cq->txq;
962 struct mana_port_context *apc;
963 struct netdev_queue *net_txq;
964 struct gdma_queue *gdma_wq;
965 unsigned int avail_space;
966 struct net_device *ndev;
967 struct sk_buff *skb;
968 bool txq_stopped;
969 int comp_read;
970 int i;
971
972 ndev = txq->ndev;
973 apc = netdev_priv(ndev);
974
975 comp_read = mana_gd_poll_cq(cq->gdma_cq, completions,
976 CQE_POLLING_BUFFER);
977
978 if (comp_read < 1)
979 return;
980
981 for (i = 0; i < comp_read; i++) {
982 struct mana_tx_comp_oob *cqe_oob;
983
984 if (WARN_ON_ONCE(!completions[i].is_sq))
985 return;
986
987 cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data;
988 if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type !=
989 MANA_CQE_COMPLETION))
990 return;
991
992 switch (cqe_oob->cqe_hdr.cqe_type) {
993 case CQE_TX_OKAY:
994 break;
995
996 case CQE_TX_SA_DROP:
997 case CQE_TX_MTU_DROP:
998 case CQE_TX_INVALID_OOB:
999 case CQE_TX_INVALID_ETH_TYPE:
1000 case CQE_TX_HDR_PROCESSING_ERROR:
1001 case CQE_TX_VF_DISABLED:
1002 case CQE_TX_VPORT_IDX_OUT_OF_RANGE:
1003 case CQE_TX_VPORT_DISABLED:
1004 case CQE_TX_VLAN_TAGGING_VIOLATION:
1005 WARN_ONCE(1, "TX: CQE error %d: ignored.\n",
1006 cqe_oob->cqe_hdr.cqe_type);
1007 break;
1008
1009 default:
1010 /* If the CQE type is unexpected, log an error, assert,
1011 * and go through the error path.
1012 */
1013 WARN_ONCE(1, "TX: Unexpected CQE type %d: HW BUG?\n",
1014 cqe_oob->cqe_hdr.cqe_type);
1015 return;
1016 }
1017
1018 if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num))
1019 return;
1020
1021 skb = skb_dequeue(&txq->pending_skbs);
1022 if (WARN_ON_ONCE(!skb))
1023 return;
1024
1025 wqe_info = (struct gdma_posted_wqe_info *)skb->cb;
1026 wqe_unit_cnt += wqe_info->wqe_size_in_bu;
1027
1028 mana_unmap_skb(skb, apc);
1029
1030 napi_consume_skb(skb, cq->budget);
1031
1032 pkt_transmitted++;
1033 }
1034
1035 if (WARN_ON_ONCE(wqe_unit_cnt == 0))
1036 return;
1037
1038 mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt);
1039
1040 gdma_wq = txq->gdma_sq;
1041 avail_space = mana_gd_wq_avail_space(gdma_wq);
1042
1043 /* Ensure tail updated before checking q stop */
1044 smp_mb();
1045
1046 net_txq = txq->net_txq;
1047 txq_stopped = netif_tx_queue_stopped(net_txq);
1048
1049 /* Ensure checking txq_stopped before apc->port_is_up. */
1050 smp_rmb();
1051
1052 if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1053 netif_tx_wake_queue(net_txq);
1054 apc->eth_stats.wake_queue++;
1055 }
1056
1057 if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)
1058 WARN_ON_ONCE(1);
1059
1060 cq->work_done = pkt_transmitted;
1061 }
1062
mana_post_pkt_rxq(struct mana_rxq * rxq)1063 static void mana_post_pkt_rxq(struct mana_rxq *rxq)
1064 {
1065 struct mana_recv_buf_oob *recv_buf_oob;
1066 u32 curr_index;
1067 int err;
1068
1069 curr_index = rxq->buf_index++;
1070 if (rxq->buf_index == rxq->num_rx_buf)
1071 rxq->buf_index = 0;
1072
1073 recv_buf_oob = &rxq->rx_oobs[curr_index];
1074
1075 err = mana_gd_post_and_ring(rxq->gdma_rq, &recv_buf_oob->wqe_req,
1076 &recv_buf_oob->wqe_inf);
1077 if (WARN_ON_ONCE(err))
1078 return;
1079
1080 WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1);
1081 }
1082
mana_build_skb(void * buf_va,uint pkt_len,struct xdp_buff * xdp)1083 static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len,
1084 struct xdp_buff *xdp)
1085 {
1086 struct sk_buff *skb = build_skb(buf_va, PAGE_SIZE);
1087
1088 if (!skb)
1089 return NULL;
1090
1091 if (xdp->data_hard_start) {
1092 skb_reserve(skb, xdp->data - xdp->data_hard_start);
1093 skb_put(skb, xdp->data_end - xdp->data);
1094 } else {
1095 skb_reserve(skb, XDP_PACKET_HEADROOM);
1096 skb_put(skb, pkt_len);
1097 }
1098
1099 return skb;
1100 }
1101
mana_rx_skb(void * buf_va,struct mana_rxcomp_oob * cqe,struct mana_rxq * rxq)1102 static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe,
1103 struct mana_rxq *rxq)
1104 {
1105 struct mana_stats_rx *rx_stats = &rxq->stats;
1106 struct net_device *ndev = rxq->ndev;
1107 uint pkt_len = cqe->ppi[0].pkt_len;
1108 u16 rxq_idx = rxq->rxq_idx;
1109 struct napi_struct *napi;
1110 struct xdp_buff xdp = {};
1111 struct sk_buff *skb;
1112 u32 hash_value;
1113 u32 act;
1114
1115 rxq->rx_cq.work_done++;
1116 napi = &rxq->rx_cq.napi;
1117
1118 if (!buf_va) {
1119 ++ndev->stats.rx_dropped;
1120 return;
1121 }
1122
1123 act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len);
1124
1125 if (act == XDP_REDIRECT && !rxq->xdp_rc)
1126 return;
1127
1128 if (act != XDP_PASS && act != XDP_TX)
1129 goto drop_xdp;
1130
1131 skb = mana_build_skb(buf_va, pkt_len, &xdp);
1132
1133 if (!skb)
1134 goto drop;
1135
1136 skb->dev = napi->dev;
1137
1138 skb->protocol = eth_type_trans(skb, ndev);
1139 skb_checksum_none_assert(skb);
1140 skb_record_rx_queue(skb, rxq_idx);
1141
1142 if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) {
1143 if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed)
1144 skb->ip_summed = CHECKSUM_UNNECESSARY;
1145 }
1146
1147 if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
1148 hash_value = cqe->ppi[0].pkt_hash;
1149
1150 if (cqe->rx_hashtype & MANA_HASH_L4)
1151 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
1152 else
1153 skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3);
1154 }
1155
1156 u64_stats_update_begin(&rx_stats->syncp);
1157 rx_stats->packets++;
1158 rx_stats->bytes += pkt_len;
1159
1160 if (act == XDP_TX)
1161 rx_stats->xdp_tx++;
1162 u64_stats_update_end(&rx_stats->syncp);
1163
1164 if (act == XDP_TX) {
1165 skb_set_queue_mapping(skb, rxq_idx);
1166 mana_xdp_tx(skb, ndev);
1167 return;
1168 }
1169
1170 napi_gro_receive(napi, skb);
1171
1172 return;
1173
1174 drop_xdp:
1175 u64_stats_update_begin(&rx_stats->syncp);
1176 rx_stats->xdp_drop++;
1177 u64_stats_update_end(&rx_stats->syncp);
1178
1179 drop:
1180 WARN_ON_ONCE(rxq->xdp_save_page);
1181 rxq->xdp_save_page = virt_to_page(buf_va);
1182
1183 ++ndev->stats.rx_dropped;
1184
1185 return;
1186 }
1187
mana_process_rx_cqe(struct mana_rxq * rxq,struct mana_cq * cq,struct gdma_comp * cqe)1188 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
1189 struct gdma_comp *cqe)
1190 {
1191 struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data;
1192 struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
1193 struct net_device *ndev = rxq->ndev;
1194 struct mana_recv_buf_oob *rxbuf_oob;
1195 struct device *dev = gc->dev;
1196 void *new_buf, *old_buf;
1197 struct page *new_page;
1198 u32 curr, pktlen;
1199 dma_addr_t da;
1200
1201 switch (oob->cqe_hdr.cqe_type) {
1202 case CQE_RX_OKAY:
1203 break;
1204
1205 case CQE_RX_TRUNCATED:
1206 ++ndev->stats.rx_dropped;
1207 rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
1208 netdev_warn_once(ndev, "Dropped a truncated packet\n");
1209 goto drop;
1210
1211 case CQE_RX_COALESCED_4:
1212 netdev_err(ndev, "RX coalescing is unsupported\n");
1213 return;
1214
1215 case CQE_RX_OBJECT_FENCE:
1216 complete(&rxq->fence_event);
1217 return;
1218
1219 default:
1220 netdev_err(ndev, "Unknown RX CQE type = %d\n",
1221 oob->cqe_hdr.cqe_type);
1222 return;
1223 }
1224
1225 pktlen = oob->ppi[0].pkt_len;
1226
1227 if (pktlen == 0) {
1228 /* data packets should never have packetlength of zero */
1229 netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
1230 rxq->gdma_id, cq->gdma_id, rxq->rxobj);
1231 return;
1232 }
1233
1234 curr = rxq->buf_index;
1235 rxbuf_oob = &rxq->rx_oobs[curr];
1236 WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
1237
1238 /* Reuse XDP dropped page if available */
1239 if (rxq->xdp_save_page) {
1240 new_page = rxq->xdp_save_page;
1241 rxq->xdp_save_page = NULL;
1242 } else {
1243 new_page = alloc_page(GFP_ATOMIC);
1244 }
1245
1246 if (new_page) {
1247 da = dma_map_page(dev, new_page, XDP_PACKET_HEADROOM, rxq->datasize,
1248 DMA_FROM_DEVICE);
1249
1250 if (dma_mapping_error(dev, da)) {
1251 __free_page(new_page);
1252 new_page = NULL;
1253 }
1254 }
1255
1256 new_buf = new_page ? page_to_virt(new_page) : NULL;
1257
1258 if (new_buf) {
1259 dma_unmap_page(dev, rxbuf_oob->buf_dma_addr, rxq->datasize,
1260 DMA_FROM_DEVICE);
1261
1262 old_buf = rxbuf_oob->buf_va;
1263
1264 /* refresh the rxbuf_oob with the new page */
1265 rxbuf_oob->buf_va = new_buf;
1266 rxbuf_oob->buf_dma_addr = da;
1267 rxbuf_oob->sgl[0].address = rxbuf_oob->buf_dma_addr;
1268 } else {
1269 old_buf = NULL; /* drop the packet if no memory */
1270 }
1271
1272 mana_rx_skb(old_buf, oob, rxq);
1273
1274 drop:
1275 mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
1276
1277 mana_post_pkt_rxq(rxq);
1278 }
1279
mana_poll_rx_cq(struct mana_cq * cq)1280 static void mana_poll_rx_cq(struct mana_cq *cq)
1281 {
1282 struct gdma_comp *comp = cq->gdma_comp_buf;
1283 struct mana_rxq *rxq = cq->rxq;
1284 int comp_read, i;
1285
1286 comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER);
1287 WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);
1288
1289 rxq->xdp_flush = false;
1290
1291 for (i = 0; i < comp_read; i++) {
1292 if (WARN_ON_ONCE(comp[i].is_sq))
1293 return;
1294
1295 /* verify recv cqe references the right rxq */
1296 if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id))
1297 return;
1298
1299 mana_process_rx_cqe(rxq, cq, &comp[i]);
1300 }
1301
1302 if (rxq->xdp_flush)
1303 xdp_do_flush();
1304 }
1305
mana_cq_handler(void * context,struct gdma_queue * gdma_queue)1306 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
1307 {
1308 struct mana_cq *cq = context;
1309 u8 arm_bit;
1310 int w;
1311
1312 WARN_ON_ONCE(cq->gdma_cq != gdma_queue);
1313
1314 if (cq->type == MANA_CQ_TYPE_RX)
1315 mana_poll_rx_cq(cq);
1316 else
1317 mana_poll_tx_cq(cq);
1318
1319 w = cq->work_done;
1320
1321 if (w < cq->budget &&
1322 napi_complete_done(&cq->napi, w)) {
1323 arm_bit = SET_ARM_BIT;
1324 } else {
1325 arm_bit = 0;
1326 }
1327
1328 mana_gd_ring_cq(gdma_queue, arm_bit);
1329
1330 return w;
1331 }
1332
mana_poll(struct napi_struct * napi,int budget)1333 static int mana_poll(struct napi_struct *napi, int budget)
1334 {
1335 struct mana_cq *cq = container_of(napi, struct mana_cq, napi);
1336 int w;
1337
1338 cq->work_done = 0;
1339 cq->budget = budget;
1340
1341 w = mana_cq_handler(cq, cq->gdma_cq);
1342
1343 return min(w, budget);
1344 }
1345
mana_schedule_napi(void * context,struct gdma_queue * gdma_queue)1346 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue)
1347 {
1348 struct mana_cq *cq = context;
1349
1350 napi_schedule_irqoff(&cq->napi);
1351 }
1352
mana_deinit_cq(struct mana_port_context * apc,struct mana_cq * cq)1353 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq)
1354 {
1355 struct gdma_dev *gd = apc->ac->gdma_dev;
1356
1357 if (!cq->gdma_cq)
1358 return;
1359
1360 mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq);
1361 }
1362
mana_deinit_txq(struct mana_port_context * apc,struct mana_txq * txq)1363 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq)
1364 {
1365 struct gdma_dev *gd = apc->ac->gdma_dev;
1366
1367 if (!txq->gdma_sq)
1368 return;
1369
1370 mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);
1371 }
1372
mana_destroy_txq(struct mana_port_context * apc)1373 static void mana_destroy_txq(struct mana_port_context *apc)
1374 {
1375 struct napi_struct *napi;
1376 int i;
1377
1378 if (!apc->tx_qp)
1379 return;
1380
1381 for (i = 0; i < apc->num_queues; i++) {
1382 napi = &apc->tx_qp[i].tx_cq.napi;
1383 napi_synchronize(napi);
1384 napi_disable(napi);
1385 netif_napi_del(napi);
1386
1387 mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
1388
1389 mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);
1390
1391 mana_deinit_txq(apc, &apc->tx_qp[i].txq);
1392 }
1393
1394 kfree(apc->tx_qp);
1395 apc->tx_qp = NULL;
1396 }
1397
mana_create_txq(struct mana_port_context * apc,struct net_device * net)1398 static int mana_create_txq(struct mana_port_context *apc,
1399 struct net_device *net)
1400 {
1401 struct mana_context *ac = apc->ac;
1402 struct gdma_dev *gd = ac->gdma_dev;
1403 struct mana_obj_spec wq_spec;
1404 struct mana_obj_spec cq_spec;
1405 struct gdma_queue_spec spec;
1406 struct gdma_context *gc;
1407 struct mana_txq *txq;
1408 struct mana_cq *cq;
1409 u32 txq_size;
1410 u32 cq_size;
1411 int err;
1412 int i;
1413
1414 apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp),
1415 GFP_KERNEL);
1416 if (!apc->tx_qp)
1417 return -ENOMEM;
1418
1419 /* The minimum size of the WQE is 32 bytes, hence
1420 * MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs
1421 * the SQ can store. This value is then used to size other queues
1422 * to prevent overflow.
1423 */
1424 txq_size = MAX_SEND_BUFFERS_PER_QUEUE * 32;
1425 BUILD_BUG_ON(!PAGE_ALIGNED(txq_size));
1426
1427 cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE;
1428 cq_size = PAGE_ALIGN(cq_size);
1429
1430 gc = gd->gdma_context;
1431
1432 for (i = 0; i < apc->num_queues; i++) {
1433 apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
1434
1435 /* Create SQ */
1436 txq = &apc->tx_qp[i].txq;
1437
1438 u64_stats_init(&txq->stats.syncp);
1439 txq->ndev = net;
1440 txq->net_txq = netdev_get_tx_queue(net, i);
1441 txq->vp_offset = apc->tx_vp_offset;
1442 skb_queue_head_init(&txq->pending_skbs);
1443
1444 memset(&spec, 0, sizeof(spec));
1445 spec.type = GDMA_SQ;
1446 spec.monitor_avl_buf = true;
1447 spec.queue_size = txq_size;
1448 err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq);
1449 if (err)
1450 goto out;
1451
1452 /* Create SQ's CQ */
1453 cq = &apc->tx_qp[i].tx_cq;
1454 cq->type = MANA_CQ_TYPE_TX;
1455
1456 cq->txq = txq;
1457
1458 memset(&spec, 0, sizeof(spec));
1459 spec.type = GDMA_CQ;
1460 spec.monitor_avl_buf = false;
1461 spec.queue_size = cq_size;
1462 spec.cq.callback = mana_schedule_napi;
1463 spec.cq.parent_eq = ac->eqs[i].eq;
1464 spec.cq.context = cq;
1465 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
1466 if (err)
1467 goto out;
1468
1469 memset(&wq_spec, 0, sizeof(wq_spec));
1470 memset(&cq_spec, 0, sizeof(cq_spec));
1471
1472 wq_spec.gdma_region = txq->gdma_sq->mem_info.gdma_region;
1473 wq_spec.queue_size = txq->gdma_sq->queue_size;
1474
1475 cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region;
1476 cq_spec.queue_size = cq->gdma_cq->queue_size;
1477 cq_spec.modr_ctx_id = 0;
1478 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
1479
1480 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
1481 &wq_spec, &cq_spec,
1482 &apc->tx_qp[i].tx_object);
1483
1484 if (err)
1485 goto out;
1486
1487 txq->gdma_sq->id = wq_spec.queue_index;
1488 cq->gdma_cq->id = cq_spec.queue_index;
1489
1490 txq->gdma_sq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1491 cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1492
1493 txq->gdma_txq_id = txq->gdma_sq->id;
1494
1495 cq->gdma_id = cq->gdma_cq->id;
1496
1497 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
1498 err = -EINVAL;
1499 goto out;
1500 }
1501
1502 gc->cq_table[cq->gdma_id] = cq->gdma_cq;
1503
1504 netif_napi_add_tx(net, &cq->napi, mana_poll);
1505 napi_enable(&cq->napi);
1506
1507 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
1508 }
1509
1510 return 0;
1511 out:
1512 mana_destroy_txq(apc);
1513 return err;
1514 }
1515
mana_destroy_rxq(struct mana_port_context * apc,struct mana_rxq * rxq,bool validate_state)1516 static void mana_destroy_rxq(struct mana_port_context *apc,
1517 struct mana_rxq *rxq, bool validate_state)
1518
1519 {
1520 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1521 struct mana_recv_buf_oob *rx_oob;
1522 struct device *dev = gc->dev;
1523 struct napi_struct *napi;
1524 int i;
1525
1526 if (!rxq)
1527 return;
1528
1529 napi = &rxq->rx_cq.napi;
1530
1531 if (validate_state)
1532 napi_synchronize(napi);
1533
1534 napi_disable(napi);
1535
1536 xdp_rxq_info_unreg(&rxq->xdp_rxq);
1537
1538 netif_napi_del(napi);
1539
1540 mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
1541
1542 mana_deinit_cq(apc, &rxq->rx_cq);
1543
1544 if (rxq->xdp_save_page)
1545 __free_page(rxq->xdp_save_page);
1546
1547 for (i = 0; i < rxq->num_rx_buf; i++) {
1548 rx_oob = &rxq->rx_oobs[i];
1549
1550 if (!rx_oob->buf_va)
1551 continue;
1552
1553 dma_unmap_page(dev, rx_oob->buf_dma_addr, rxq->datasize,
1554 DMA_FROM_DEVICE);
1555
1556 free_page((unsigned long)rx_oob->buf_va);
1557 rx_oob->buf_va = NULL;
1558 }
1559
1560 if (rxq->gdma_rq)
1561 mana_gd_destroy_queue(gc, rxq->gdma_rq);
1562
1563 kfree(rxq);
1564 }
1565
1566 #define MANA_WQE_HEADER_SIZE 16
1567 #define MANA_WQE_SGE_SIZE 16
1568
mana_alloc_rx_wqe(struct mana_port_context * apc,struct mana_rxq * rxq,u32 * rxq_size,u32 * cq_size)1569 static int mana_alloc_rx_wqe(struct mana_port_context *apc,
1570 struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size)
1571 {
1572 struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1573 struct mana_recv_buf_oob *rx_oob;
1574 struct device *dev = gc->dev;
1575 struct page *page;
1576 dma_addr_t da;
1577 u32 buf_idx;
1578
1579 WARN_ON(rxq->datasize == 0 || rxq->datasize > PAGE_SIZE);
1580
1581 *rxq_size = 0;
1582 *cq_size = 0;
1583
1584 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
1585 rx_oob = &rxq->rx_oobs[buf_idx];
1586 memset(rx_oob, 0, sizeof(*rx_oob));
1587
1588 page = alloc_page(GFP_KERNEL);
1589 if (!page)
1590 return -ENOMEM;
1591
1592 da = dma_map_page(dev, page, XDP_PACKET_HEADROOM, rxq->datasize,
1593 DMA_FROM_DEVICE);
1594
1595 if (dma_mapping_error(dev, da)) {
1596 __free_page(page);
1597 return -ENOMEM;
1598 }
1599
1600 rx_oob->buf_va = page_to_virt(page);
1601 rx_oob->buf_dma_addr = da;
1602
1603 rx_oob->num_sge = 1;
1604 rx_oob->sgl[0].address = rx_oob->buf_dma_addr;
1605 rx_oob->sgl[0].size = rxq->datasize;
1606 rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey;
1607
1608 rx_oob->wqe_req.sgl = rx_oob->sgl;
1609 rx_oob->wqe_req.num_sge = rx_oob->num_sge;
1610 rx_oob->wqe_req.inline_oob_size = 0;
1611 rx_oob->wqe_req.inline_oob_data = NULL;
1612 rx_oob->wqe_req.flags = 0;
1613 rx_oob->wqe_req.client_data_unit = 0;
1614
1615 *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE +
1616 MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32);
1617 *cq_size += COMP_ENTRY_SIZE;
1618 }
1619
1620 return 0;
1621 }
1622
mana_push_wqe(struct mana_rxq * rxq)1623 static int mana_push_wqe(struct mana_rxq *rxq)
1624 {
1625 struct mana_recv_buf_oob *rx_oob;
1626 u32 buf_idx;
1627 int err;
1628
1629 for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
1630 rx_oob = &rxq->rx_oobs[buf_idx];
1631
1632 err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req,
1633 &rx_oob->wqe_inf);
1634 if (err)
1635 return -ENOSPC;
1636 }
1637
1638 return 0;
1639 }
1640
mana_create_rxq(struct mana_port_context * apc,u32 rxq_idx,struct mana_eq * eq,struct net_device * ndev)1641 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
1642 u32 rxq_idx, struct mana_eq *eq,
1643 struct net_device *ndev)
1644 {
1645 struct gdma_dev *gd = apc->ac->gdma_dev;
1646 struct mana_obj_spec wq_spec;
1647 struct mana_obj_spec cq_spec;
1648 struct gdma_queue_spec spec;
1649 struct mana_cq *cq = NULL;
1650 struct gdma_context *gc;
1651 u32 cq_size, rq_size;
1652 struct mana_rxq *rxq;
1653 int err;
1654
1655 gc = gd->gdma_context;
1656
1657 rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE),
1658 GFP_KERNEL);
1659 if (!rxq)
1660 return NULL;
1661
1662 rxq->ndev = ndev;
1663 rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
1664 rxq->rxq_idx = rxq_idx;
1665 rxq->datasize = ALIGN(MAX_FRAME_SIZE, 64);
1666 rxq->rxobj = INVALID_MANA_HANDLE;
1667
1668 err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size);
1669 if (err)
1670 goto out;
1671
1672 rq_size = PAGE_ALIGN(rq_size);
1673 cq_size = PAGE_ALIGN(cq_size);
1674
1675 /* Create RQ */
1676 memset(&spec, 0, sizeof(spec));
1677 spec.type = GDMA_RQ;
1678 spec.monitor_avl_buf = true;
1679 spec.queue_size = rq_size;
1680 err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq);
1681 if (err)
1682 goto out;
1683
1684 /* Create RQ's CQ */
1685 cq = &rxq->rx_cq;
1686 cq->type = MANA_CQ_TYPE_RX;
1687 cq->rxq = rxq;
1688
1689 memset(&spec, 0, sizeof(spec));
1690 spec.type = GDMA_CQ;
1691 spec.monitor_avl_buf = false;
1692 spec.queue_size = cq_size;
1693 spec.cq.callback = mana_schedule_napi;
1694 spec.cq.parent_eq = eq->eq;
1695 spec.cq.context = cq;
1696 err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
1697 if (err)
1698 goto out;
1699
1700 memset(&wq_spec, 0, sizeof(wq_spec));
1701 memset(&cq_spec, 0, sizeof(cq_spec));
1702 wq_spec.gdma_region = rxq->gdma_rq->mem_info.gdma_region;
1703 wq_spec.queue_size = rxq->gdma_rq->queue_size;
1704
1705 cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region;
1706 cq_spec.queue_size = cq->gdma_cq->queue_size;
1707 cq_spec.modr_ctx_id = 0;
1708 cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
1709
1710 err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
1711 &wq_spec, &cq_spec, &rxq->rxobj);
1712 if (err)
1713 goto out;
1714
1715 rxq->gdma_rq->id = wq_spec.queue_index;
1716 cq->gdma_cq->id = cq_spec.queue_index;
1717
1718 rxq->gdma_rq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1719 cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1720
1721 rxq->gdma_id = rxq->gdma_rq->id;
1722 cq->gdma_id = cq->gdma_cq->id;
1723
1724 err = mana_push_wqe(rxq);
1725 if (err)
1726 goto out;
1727
1728 if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
1729 err = -EINVAL;
1730 goto out;
1731 }
1732
1733 gc->cq_table[cq->gdma_id] = cq->gdma_cq;
1734
1735 netif_napi_add_weight(ndev, &cq->napi, mana_poll, 1);
1736
1737 WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx,
1738 cq->napi.napi_id));
1739 WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq,
1740 MEM_TYPE_PAGE_SHARED, NULL));
1741
1742 napi_enable(&cq->napi);
1743
1744 mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
1745 out:
1746 if (!err)
1747 return rxq;
1748
1749 netdev_err(ndev, "Failed to create RXQ: err = %d\n", err);
1750
1751 mana_destroy_rxq(apc, rxq, false);
1752
1753 if (cq)
1754 mana_deinit_cq(apc, cq);
1755
1756 return NULL;
1757 }
1758
mana_add_rx_queues(struct mana_port_context * apc,struct net_device * ndev)1759 static int mana_add_rx_queues(struct mana_port_context *apc,
1760 struct net_device *ndev)
1761 {
1762 struct mana_context *ac = apc->ac;
1763 struct mana_rxq *rxq;
1764 int err = 0;
1765 int i;
1766
1767 for (i = 0; i < apc->num_queues; i++) {
1768 rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev);
1769 if (!rxq) {
1770 err = -ENOMEM;
1771 goto out;
1772 }
1773
1774 u64_stats_init(&rxq->stats.syncp);
1775
1776 apc->rxqs[i] = rxq;
1777 }
1778
1779 apc->default_rxobj = apc->rxqs[0]->rxobj;
1780 out:
1781 return err;
1782 }
1783
mana_destroy_vport(struct mana_port_context * apc)1784 static void mana_destroy_vport(struct mana_port_context *apc)
1785 {
1786 struct gdma_dev *gd = apc->ac->gdma_dev;
1787 struct mana_rxq *rxq;
1788 u32 rxq_idx;
1789
1790 for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
1791 rxq = apc->rxqs[rxq_idx];
1792 if (!rxq)
1793 continue;
1794
1795 mana_destroy_rxq(apc, rxq, true);
1796 apc->rxqs[rxq_idx] = NULL;
1797 }
1798
1799 mana_destroy_txq(apc);
1800
1801 if (gd->gdma_context->is_pf)
1802 mana_pf_deregister_hw_vport(apc);
1803 }
1804
mana_create_vport(struct mana_port_context * apc,struct net_device * net)1805 static int mana_create_vport(struct mana_port_context *apc,
1806 struct net_device *net)
1807 {
1808 struct gdma_dev *gd = apc->ac->gdma_dev;
1809 int err;
1810
1811 apc->default_rxobj = INVALID_MANA_HANDLE;
1812
1813 if (gd->gdma_context->is_pf) {
1814 err = mana_pf_register_hw_vport(apc);
1815 if (err)
1816 return err;
1817 }
1818
1819 err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);
1820 if (err)
1821 return err;
1822
1823 return mana_create_txq(apc, net);
1824 }
1825
mana_rss_table_init(struct mana_port_context * apc)1826 static void mana_rss_table_init(struct mana_port_context *apc)
1827 {
1828 int i;
1829
1830 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++)
1831 apc->indir_table[i] =
1832 ethtool_rxfh_indir_default(i, apc->num_queues);
1833 }
1834
mana_config_rss(struct mana_port_context * apc,enum TRI_STATE rx,bool update_hash,bool update_tab)1835 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx,
1836 bool update_hash, bool update_tab)
1837 {
1838 u32 queue_idx;
1839 int err;
1840 int i;
1841
1842 if (update_tab) {
1843 for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) {
1844 queue_idx = apc->indir_table[i];
1845 apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;
1846 }
1847 }
1848
1849 err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab);
1850 if (err)
1851 return err;
1852
1853 mana_fence_rqs(apc);
1854
1855 return 0;
1856 }
1857
mana_init_port(struct net_device * ndev)1858 static int mana_init_port(struct net_device *ndev)
1859 {
1860 struct mana_port_context *apc = netdev_priv(ndev);
1861 u32 max_txq, max_rxq, max_queues;
1862 int port_idx = apc->port_idx;
1863 u32 num_indirect_entries;
1864 int err;
1865
1866 err = mana_init_port_context(apc);
1867 if (err)
1868 return err;
1869
1870 err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq,
1871 &num_indirect_entries);
1872 if (err) {
1873 netdev_err(ndev, "Failed to query info for vPort %d\n",
1874 port_idx);
1875 goto reset_apc;
1876 }
1877
1878 max_queues = min_t(u32, max_txq, max_rxq);
1879 if (apc->max_queues > max_queues)
1880 apc->max_queues = max_queues;
1881
1882 if (apc->num_queues > apc->max_queues)
1883 apc->num_queues = apc->max_queues;
1884
1885 eth_hw_addr_set(ndev, apc->mac_addr);
1886
1887 return 0;
1888
1889 reset_apc:
1890 kfree(apc->rxqs);
1891 apc->rxqs = NULL;
1892 return err;
1893 }
1894
mana_alloc_queues(struct net_device * ndev)1895 int mana_alloc_queues(struct net_device *ndev)
1896 {
1897 struct mana_port_context *apc = netdev_priv(ndev);
1898 struct gdma_dev *gd = apc->ac->gdma_dev;
1899 int err;
1900
1901 err = mana_create_vport(apc, ndev);
1902 if (err)
1903 return err;
1904
1905 err = netif_set_real_num_tx_queues(ndev, apc->num_queues);
1906 if (err)
1907 goto destroy_vport;
1908
1909 err = mana_add_rx_queues(apc, ndev);
1910 if (err)
1911 goto destroy_vport;
1912
1913 apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE;
1914
1915 err = netif_set_real_num_rx_queues(ndev, apc->num_queues);
1916 if (err)
1917 goto destroy_vport;
1918
1919 mana_rss_table_init(apc);
1920
1921 err = mana_config_rss(apc, TRI_STATE_TRUE, true, true);
1922 if (err)
1923 goto destroy_vport;
1924
1925 if (gd->gdma_context->is_pf) {
1926 err = mana_pf_register_filter(apc);
1927 if (err)
1928 goto destroy_vport;
1929 }
1930
1931 mana_chn_setxdp(apc, mana_xdp_get(apc));
1932
1933 return 0;
1934
1935 destroy_vport:
1936 mana_destroy_vport(apc);
1937 return err;
1938 }
1939
mana_attach(struct net_device * ndev)1940 int mana_attach(struct net_device *ndev)
1941 {
1942 struct mana_port_context *apc = netdev_priv(ndev);
1943 int err;
1944
1945 ASSERT_RTNL();
1946
1947 err = mana_init_port(ndev);
1948 if (err)
1949 return err;
1950
1951 if (apc->port_st_save) {
1952 err = mana_alloc_queues(ndev);
1953 if (err) {
1954 mana_cleanup_port_context(apc);
1955 return err;
1956 }
1957 }
1958
1959 apc->port_is_up = apc->port_st_save;
1960
1961 /* Ensure port state updated before txq state */
1962 smp_wmb();
1963
1964 if (apc->port_is_up)
1965 netif_carrier_on(ndev);
1966
1967 netif_device_attach(ndev);
1968
1969 return 0;
1970 }
1971
mana_dealloc_queues(struct net_device * ndev)1972 static int mana_dealloc_queues(struct net_device *ndev)
1973 {
1974 struct mana_port_context *apc = netdev_priv(ndev);
1975 struct gdma_dev *gd = apc->ac->gdma_dev;
1976 struct mana_txq *txq;
1977 int i, err;
1978
1979 if (apc->port_is_up)
1980 return -EINVAL;
1981
1982 mana_chn_setxdp(apc, NULL);
1983
1984 if (gd->gdma_context->is_pf)
1985 mana_pf_deregister_filter(apc);
1986
1987 /* No packet can be transmitted now since apc->port_is_up is false.
1988 * There is still a tiny chance that mana_poll_tx_cq() can re-enable
1989 * a txq because it may not timely see apc->port_is_up being cleared
1990 * to false, but it doesn't matter since mana_start_xmit() drops any
1991 * new packets due to apc->port_is_up being false.
1992 *
1993 * Drain all the in-flight TX packets
1994 */
1995 for (i = 0; i < apc->num_queues; i++) {
1996 txq = &apc->tx_qp[i].txq;
1997
1998 while (atomic_read(&txq->pending_sends) > 0)
1999 usleep_range(1000, 2000);
2000 }
2001
2002 /* We're 100% sure the queues can no longer be woken up, because
2003 * we're sure now mana_poll_tx_cq() can't be running.
2004 */
2005
2006 apc->rss_state = TRI_STATE_FALSE;
2007 err = mana_config_rss(apc, TRI_STATE_FALSE, false, false);
2008 if (err) {
2009 netdev_err(ndev, "Failed to disable vPort: %d\n", err);
2010 return err;
2011 }
2012
2013 mana_destroy_vport(apc);
2014
2015 return 0;
2016 }
2017
mana_detach(struct net_device * ndev,bool from_close)2018 int mana_detach(struct net_device *ndev, bool from_close)
2019 {
2020 struct mana_port_context *apc = netdev_priv(ndev);
2021 int err;
2022
2023 ASSERT_RTNL();
2024
2025 apc->port_st_save = apc->port_is_up;
2026 apc->port_is_up = false;
2027
2028 /* Ensure port state updated before txq state */
2029 smp_wmb();
2030
2031 netif_tx_disable(ndev);
2032 netif_carrier_off(ndev);
2033
2034 if (apc->port_st_save) {
2035 err = mana_dealloc_queues(ndev);
2036 if (err)
2037 return err;
2038 }
2039
2040 if (!from_close) {
2041 netif_device_detach(ndev);
2042 mana_cleanup_port_context(apc);
2043 }
2044
2045 return 0;
2046 }
2047
mana_probe_port(struct mana_context * ac,int port_idx,struct net_device ** ndev_storage)2048 static int mana_probe_port(struct mana_context *ac, int port_idx,
2049 struct net_device **ndev_storage)
2050 {
2051 struct gdma_context *gc = ac->gdma_dev->gdma_context;
2052 struct mana_port_context *apc;
2053 struct net_device *ndev;
2054 int err;
2055
2056 ndev = alloc_etherdev_mq(sizeof(struct mana_port_context),
2057 gc->max_num_queues);
2058 if (!ndev)
2059 return -ENOMEM;
2060
2061 *ndev_storage = ndev;
2062
2063 apc = netdev_priv(ndev);
2064 apc->ac = ac;
2065 apc->ndev = ndev;
2066 apc->max_queues = gc->max_num_queues;
2067 apc->num_queues = gc->max_num_queues;
2068 apc->port_handle = INVALID_MANA_HANDLE;
2069 apc->pf_filter_handle = INVALID_MANA_HANDLE;
2070 apc->port_idx = port_idx;
2071
2072 ndev->netdev_ops = &mana_devops;
2073 ndev->ethtool_ops = &mana_ethtool_ops;
2074 ndev->mtu = ETH_DATA_LEN;
2075 ndev->max_mtu = ndev->mtu;
2076 ndev->min_mtu = ndev->mtu;
2077 ndev->needed_headroom = MANA_HEADROOM;
2078 SET_NETDEV_DEV(ndev, gc->dev);
2079
2080 netif_carrier_off(ndev);
2081
2082 netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE);
2083
2084 err = mana_init_port(ndev);
2085 if (err)
2086 goto free_net;
2087
2088 netdev_lockdep_set_classes(ndev);
2089
2090 ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2091 ndev->hw_features |= NETIF_F_RXCSUM;
2092 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2093 ndev->hw_features |= NETIF_F_RXHASH;
2094 ndev->features = ndev->hw_features;
2095 ndev->vlan_features = 0;
2096
2097 err = register_netdev(ndev);
2098 if (err) {
2099 netdev_err(ndev, "Unable to register netdev.\n");
2100 goto reset_apc;
2101 }
2102
2103 return 0;
2104
2105 reset_apc:
2106 kfree(apc->rxqs);
2107 apc->rxqs = NULL;
2108 free_net:
2109 *ndev_storage = NULL;
2110 netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err);
2111 free_netdev(ndev);
2112 return err;
2113 }
2114
mana_probe(struct gdma_dev * gd,bool resuming)2115 int mana_probe(struct gdma_dev *gd, bool resuming)
2116 {
2117 struct gdma_context *gc = gd->gdma_context;
2118 struct mana_context *ac = gd->driver_data;
2119 struct device *dev = gc->dev;
2120 u16 num_ports = 0;
2121 int err;
2122 int i;
2123
2124 dev_info(dev,
2125 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n",
2126 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION);
2127
2128 err = mana_gd_register_device(gd);
2129 if (err)
2130 return err;
2131
2132 if (!resuming) {
2133 ac = kzalloc(sizeof(*ac), GFP_KERNEL);
2134 if (!ac)
2135 return -ENOMEM;
2136
2137 ac->gdma_dev = gd;
2138 gd->driver_data = ac;
2139 }
2140
2141 err = mana_create_eq(ac);
2142 if (err)
2143 goto out;
2144
2145 err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION,
2146 MANA_MICRO_VERSION, &num_ports);
2147 if (err)
2148 goto out;
2149
2150 if (!resuming) {
2151 ac->num_ports = num_ports;
2152 } else {
2153 if (ac->num_ports != num_ports) {
2154 dev_err(dev, "The number of vPorts changed: %d->%d\n",
2155 ac->num_ports, num_ports);
2156 err = -EPROTO;
2157 goto out;
2158 }
2159 }
2160
2161 if (ac->num_ports == 0)
2162 dev_err(dev, "Failed to detect any vPort\n");
2163
2164 if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)
2165 ac->num_ports = MAX_PORTS_IN_MANA_DEV;
2166
2167 if (!resuming) {
2168 for (i = 0; i < ac->num_ports; i++) {
2169 err = mana_probe_port(ac, i, &ac->ports[i]);
2170 if (err)
2171 break;
2172 }
2173 } else {
2174 for (i = 0; i < ac->num_ports; i++) {
2175 rtnl_lock();
2176 err = mana_attach(ac->ports[i]);
2177 rtnl_unlock();
2178 if (err)
2179 break;
2180 }
2181 }
2182 out:
2183 if (err)
2184 mana_remove(gd, false);
2185
2186 return err;
2187 }
2188
mana_remove(struct gdma_dev * gd,bool suspending)2189 void mana_remove(struct gdma_dev *gd, bool suspending)
2190 {
2191 struct gdma_context *gc = gd->gdma_context;
2192 struct mana_context *ac = gd->driver_data;
2193 struct device *dev = gc->dev;
2194 struct net_device *ndev;
2195 int err;
2196 int i;
2197
2198 for (i = 0; i < ac->num_ports; i++) {
2199 ndev = ac->ports[i];
2200 if (!ndev) {
2201 if (i == 0)
2202 dev_err(dev, "No net device to remove\n");
2203 goto out;
2204 }
2205
2206 /* All cleanup actions should stay after rtnl_lock(), otherwise
2207 * other functions may access partially cleaned up data.
2208 */
2209 rtnl_lock();
2210
2211 err = mana_detach(ndev, false);
2212 if (err)
2213 netdev_err(ndev, "Failed to detach vPort %d: %d\n",
2214 i, err);
2215
2216 if (suspending) {
2217 /* No need to unregister the ndev. */
2218 rtnl_unlock();
2219 continue;
2220 }
2221
2222 unregister_netdevice(ndev);
2223
2224 rtnl_unlock();
2225
2226 free_netdev(ndev);
2227 }
2228
2229 mana_destroy_eq(ac);
2230
2231 out:
2232 mana_gd_deregister_device(gd);
2233
2234 if (suspending)
2235 return;
2236
2237 gd->driver_data = NULL;
2238 gd->gdma_context = NULL;
2239 kfree(ac);
2240 }
2241