1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
2 /* Copyright (c) 2021, Microsoft Corporation. */
3 
4 #include <uapi/linux/bpf.h>
5 
6 #include <linux/inetdevice.h>
7 #include <linux/etherdevice.h>
8 #include <linux/ethtool.h>
9 #include <linux/filter.h>
10 #include <linux/mm.h>
11 
12 #include <net/checksum.h>
13 #include <net/ip6_checksum.h>
14 
15 #include "mana.h"
16 
17 /* Microsoft Azure Network Adapter (MANA) functions */
18 
mana_open(struct net_device * ndev)19 static int mana_open(struct net_device *ndev)
20 {
21 	struct mana_port_context *apc = netdev_priv(ndev);
22 	int err;
23 
24 	err = mana_alloc_queues(ndev);
25 	if (err)
26 		return err;
27 
28 	apc->port_is_up = true;
29 
30 	/* Ensure port state updated before txq state */
31 	smp_wmb();
32 
33 	netif_carrier_on(ndev);
34 	netif_tx_wake_all_queues(ndev);
35 
36 	return 0;
37 }
38 
mana_close(struct net_device * ndev)39 static int mana_close(struct net_device *ndev)
40 {
41 	struct mana_port_context *apc = netdev_priv(ndev);
42 
43 	if (!apc->port_is_up)
44 		return 0;
45 
46 	return mana_detach(ndev, true);
47 }
48 
mana_can_tx(struct gdma_queue * wq)49 static bool mana_can_tx(struct gdma_queue *wq)
50 {
51 	return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE;
52 }
53 
mana_checksum_info(struct sk_buff * skb)54 static unsigned int mana_checksum_info(struct sk_buff *skb)
55 {
56 	if (skb->protocol == htons(ETH_P_IP)) {
57 		struct iphdr *ip = ip_hdr(skb);
58 
59 		if (ip->protocol == IPPROTO_TCP)
60 			return IPPROTO_TCP;
61 
62 		if (ip->protocol == IPPROTO_UDP)
63 			return IPPROTO_UDP;
64 	} else if (skb->protocol == htons(ETH_P_IPV6)) {
65 		struct ipv6hdr *ip6 = ipv6_hdr(skb);
66 
67 		if (ip6->nexthdr == IPPROTO_TCP)
68 			return IPPROTO_TCP;
69 
70 		if (ip6->nexthdr == IPPROTO_UDP)
71 			return IPPROTO_UDP;
72 	}
73 
74 	/* No csum offloading */
75 	return 0;
76 }
77 
mana_map_skb(struct sk_buff * skb,struct mana_port_context * apc,struct mana_tx_package * tp)78 static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc,
79 			struct mana_tx_package *tp)
80 {
81 	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
82 	struct gdma_dev *gd = apc->ac->gdma_dev;
83 	struct gdma_context *gc;
84 	struct device *dev;
85 	skb_frag_t *frag;
86 	dma_addr_t da;
87 	int i;
88 
89 	gc = gd->gdma_context;
90 	dev = gc->dev;
91 	da = dma_map_single(dev, skb->data, skb_headlen(skb), DMA_TO_DEVICE);
92 
93 	if (dma_mapping_error(dev, da))
94 		return -ENOMEM;
95 
96 	ash->dma_handle[0] = da;
97 	ash->size[0] = skb_headlen(skb);
98 
99 	tp->wqe_req.sgl[0].address = ash->dma_handle[0];
100 	tp->wqe_req.sgl[0].mem_key = gd->gpa_mkey;
101 	tp->wqe_req.sgl[0].size = ash->size[0];
102 
103 	for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
104 		frag = &skb_shinfo(skb)->frags[i];
105 		da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag),
106 				      DMA_TO_DEVICE);
107 
108 		if (dma_mapping_error(dev, da))
109 			goto frag_err;
110 
111 		ash->dma_handle[i + 1] = da;
112 		ash->size[i + 1] = skb_frag_size(frag);
113 
114 		tp->wqe_req.sgl[i + 1].address = ash->dma_handle[i + 1];
115 		tp->wqe_req.sgl[i + 1].mem_key = gd->gpa_mkey;
116 		tp->wqe_req.sgl[i + 1].size = ash->size[i + 1];
117 	}
118 
119 	return 0;
120 
121 frag_err:
122 	for (i = i - 1; i >= 0; i--)
123 		dma_unmap_page(dev, ash->dma_handle[i + 1], ash->size[i + 1],
124 			       DMA_TO_DEVICE);
125 
126 	dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE);
127 
128 	return -ENOMEM;
129 }
130 
mana_start_xmit(struct sk_buff * skb,struct net_device * ndev)131 int mana_start_xmit(struct sk_buff *skb, struct net_device *ndev)
132 {
133 	enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT;
134 	struct mana_port_context *apc = netdev_priv(ndev);
135 	u16 txq_idx = skb_get_queue_mapping(skb);
136 	struct gdma_dev *gd = apc->ac->gdma_dev;
137 	bool ipv4 = false, ipv6 = false;
138 	struct mana_tx_package pkg = {};
139 	struct netdev_queue *net_txq;
140 	struct mana_stats_tx *tx_stats;
141 	struct gdma_queue *gdma_sq;
142 	unsigned int csum_type;
143 	struct mana_txq *txq;
144 	struct mana_cq *cq;
145 	int err, len;
146 
147 	if (unlikely(!apc->port_is_up))
148 		goto tx_drop;
149 
150 	if (skb_cow_head(skb, MANA_HEADROOM))
151 		goto tx_drop_count;
152 
153 	txq = &apc->tx_qp[txq_idx].txq;
154 	gdma_sq = txq->gdma_sq;
155 	cq = &apc->tx_qp[txq_idx].tx_cq;
156 
157 	pkg.tx_oob.s_oob.vcq_num = cq->gdma_id;
158 	pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame;
159 
160 	if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) {
161 		pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset;
162 		pkt_fmt = MANA_LONG_PKT_FMT;
163 	} else {
164 		pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset;
165 	}
166 
167 	pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt;
168 
169 	if (pkt_fmt == MANA_SHORT_PKT_FMT)
170 		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob);
171 	else
172 		pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob);
173 
174 	pkg.wqe_req.inline_oob_data = &pkg.tx_oob;
175 	pkg.wqe_req.flags = 0;
176 	pkg.wqe_req.client_data_unit = 0;
177 
178 	pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags;
179 	WARN_ON_ONCE(pkg.wqe_req.num_sge > 30);
180 
181 	if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) {
182 		pkg.wqe_req.sgl = pkg.sgl_array;
183 	} else {
184 		pkg.sgl_ptr = kmalloc_array(pkg.wqe_req.num_sge,
185 					    sizeof(struct gdma_sge),
186 					    GFP_ATOMIC);
187 		if (!pkg.sgl_ptr)
188 			goto tx_drop_count;
189 
190 		pkg.wqe_req.sgl = pkg.sgl_ptr;
191 	}
192 
193 	if (skb->protocol == htons(ETH_P_IP))
194 		ipv4 = true;
195 	else if (skb->protocol == htons(ETH_P_IPV6))
196 		ipv6 = true;
197 
198 	if (skb_is_gso(skb)) {
199 		pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
200 		pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
201 
202 		pkg.tx_oob.s_oob.comp_iphdr_csum = 1;
203 		pkg.tx_oob.s_oob.comp_tcp_csum = 1;
204 		pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
205 
206 		pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size;
207 		pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0;
208 		if (ipv4) {
209 			ip_hdr(skb)->tot_len = 0;
210 			ip_hdr(skb)->check = 0;
211 			tcp_hdr(skb)->check =
212 				~csum_tcpudp_magic(ip_hdr(skb)->saddr,
213 						   ip_hdr(skb)->daddr, 0,
214 						   IPPROTO_TCP, 0);
215 		} else {
216 			ipv6_hdr(skb)->payload_len = 0;
217 			tcp_hdr(skb)->check =
218 				~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
219 						 &ipv6_hdr(skb)->daddr, 0,
220 						 IPPROTO_TCP, 0);
221 		}
222 	} else if (skb->ip_summed == CHECKSUM_PARTIAL) {
223 		csum_type = mana_checksum_info(skb);
224 
225 		if (csum_type == IPPROTO_TCP) {
226 			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
227 			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
228 
229 			pkg.tx_oob.s_oob.comp_tcp_csum = 1;
230 			pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb);
231 
232 		} else if (csum_type == IPPROTO_UDP) {
233 			pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4;
234 			pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6;
235 
236 			pkg.tx_oob.s_oob.comp_udp_csum = 1;
237 		} else {
238 			/* Can't do offload of this type of checksum */
239 			if (skb_checksum_help(skb))
240 				goto free_sgl_ptr;
241 		}
242 	}
243 
244 	if (mana_map_skb(skb, apc, &pkg))
245 		goto free_sgl_ptr;
246 
247 	skb_queue_tail(&txq->pending_skbs, skb);
248 
249 	len = skb->len;
250 	net_txq = netdev_get_tx_queue(ndev, txq_idx);
251 
252 	err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req,
253 					(struct gdma_posted_wqe_info *)skb->cb);
254 	if (!mana_can_tx(gdma_sq)) {
255 		netif_tx_stop_queue(net_txq);
256 		apc->eth_stats.stop_queue++;
257 	}
258 
259 	if (err) {
260 		(void)skb_dequeue_tail(&txq->pending_skbs);
261 		netdev_warn(ndev, "Failed to post TX OOB: %d\n", err);
262 		err = NETDEV_TX_BUSY;
263 		goto tx_busy;
264 	}
265 
266 	err = NETDEV_TX_OK;
267 	atomic_inc(&txq->pending_sends);
268 
269 	mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq);
270 
271 	/* skb may be freed after mana_gd_post_work_request. Do not use it. */
272 	skb = NULL;
273 
274 	tx_stats = &txq->stats;
275 	u64_stats_update_begin(&tx_stats->syncp);
276 	tx_stats->packets++;
277 	tx_stats->bytes += len;
278 	u64_stats_update_end(&tx_stats->syncp);
279 
280 tx_busy:
281 	if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) {
282 		netif_tx_wake_queue(net_txq);
283 		apc->eth_stats.wake_queue++;
284 	}
285 
286 	kfree(pkg.sgl_ptr);
287 	return err;
288 
289 free_sgl_ptr:
290 	kfree(pkg.sgl_ptr);
291 tx_drop_count:
292 	ndev->stats.tx_dropped++;
293 tx_drop:
294 	dev_kfree_skb_any(skb);
295 	return NETDEV_TX_OK;
296 }
297 
mana_get_stats64(struct net_device * ndev,struct rtnl_link_stats64 * st)298 static void mana_get_stats64(struct net_device *ndev,
299 			     struct rtnl_link_stats64 *st)
300 {
301 	struct mana_port_context *apc = netdev_priv(ndev);
302 	unsigned int num_queues = apc->num_queues;
303 	struct mana_stats_rx *rx_stats;
304 	struct mana_stats_tx *tx_stats;
305 	unsigned int start;
306 	u64 packets, bytes;
307 	int q;
308 
309 	if (!apc->port_is_up)
310 		return;
311 
312 	netdev_stats_to_stats64(st, &ndev->stats);
313 
314 	for (q = 0; q < num_queues; q++) {
315 		rx_stats = &apc->rxqs[q]->stats;
316 
317 		do {
318 			start = u64_stats_fetch_begin_irq(&rx_stats->syncp);
319 			packets = rx_stats->packets;
320 			bytes = rx_stats->bytes;
321 		} while (u64_stats_fetch_retry_irq(&rx_stats->syncp, start));
322 
323 		st->rx_packets += packets;
324 		st->rx_bytes += bytes;
325 	}
326 
327 	for (q = 0; q < num_queues; q++) {
328 		tx_stats = &apc->tx_qp[q].txq.stats;
329 
330 		do {
331 			start = u64_stats_fetch_begin_irq(&tx_stats->syncp);
332 			packets = tx_stats->packets;
333 			bytes = tx_stats->bytes;
334 		} while (u64_stats_fetch_retry_irq(&tx_stats->syncp, start));
335 
336 		st->tx_packets += packets;
337 		st->tx_bytes += bytes;
338 	}
339 }
340 
mana_get_tx_queue(struct net_device * ndev,struct sk_buff * skb,int old_q)341 static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb,
342 			     int old_q)
343 {
344 	struct mana_port_context *apc = netdev_priv(ndev);
345 	u32 hash = skb_get_hash(skb);
346 	struct sock *sk = skb->sk;
347 	int txq;
348 
349 	txq = apc->indir_table[hash & MANA_INDIRECT_TABLE_MASK];
350 
351 	if (txq != old_q && sk && sk_fullsock(sk) &&
352 	    rcu_access_pointer(sk->sk_dst_cache))
353 		sk_tx_queue_set(sk, txq);
354 
355 	return txq;
356 }
357 
mana_select_queue(struct net_device * ndev,struct sk_buff * skb,struct net_device * sb_dev)358 static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb,
359 			     struct net_device *sb_dev)
360 {
361 	int txq;
362 
363 	if (ndev->real_num_tx_queues == 1)
364 		return 0;
365 
366 	txq = sk_tx_queue_get(skb->sk);
367 
368 	if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) {
369 		if (skb_rx_queue_recorded(skb))
370 			txq = skb_get_rx_queue(skb);
371 		else
372 			txq = mana_get_tx_queue(ndev, skb, txq);
373 	}
374 
375 	return txq;
376 }
377 
378 static const struct net_device_ops mana_devops = {
379 	.ndo_open		= mana_open,
380 	.ndo_stop		= mana_close,
381 	.ndo_select_queue	= mana_select_queue,
382 	.ndo_start_xmit		= mana_start_xmit,
383 	.ndo_validate_addr	= eth_validate_addr,
384 	.ndo_get_stats64	= mana_get_stats64,
385 	.ndo_bpf		= mana_bpf,
386 	.ndo_xdp_xmit		= mana_xdp_xmit,
387 };
388 
mana_cleanup_port_context(struct mana_port_context * apc)389 static void mana_cleanup_port_context(struct mana_port_context *apc)
390 {
391 	kfree(apc->rxqs);
392 	apc->rxqs = NULL;
393 }
394 
mana_init_port_context(struct mana_port_context * apc)395 static int mana_init_port_context(struct mana_port_context *apc)
396 {
397 	apc->rxqs = kcalloc(apc->num_queues, sizeof(struct mana_rxq *),
398 			    GFP_KERNEL);
399 
400 	return !apc->rxqs ? -ENOMEM : 0;
401 }
402 
mana_send_request(struct mana_context * ac,void * in_buf,u32 in_len,void * out_buf,u32 out_len)403 static int mana_send_request(struct mana_context *ac, void *in_buf,
404 			     u32 in_len, void *out_buf, u32 out_len)
405 {
406 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
407 	struct gdma_resp_hdr *resp = out_buf;
408 	struct gdma_req_hdr *req = in_buf;
409 	struct device *dev = gc->dev;
410 	static atomic_t activity_id;
411 	int err;
412 
413 	req->dev_id = gc->mana.dev_id;
414 	req->activity_id = atomic_inc_return(&activity_id);
415 
416 	err = mana_gd_send_request(gc, in_len, in_buf, out_len,
417 				   out_buf);
418 	if (err || resp->status) {
419 		dev_err(dev, "Failed to send mana message: %d, 0x%x\n",
420 			err, resp->status);
421 		return err ? err : -EPROTO;
422 	}
423 
424 	if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 ||
425 	    req->activity_id != resp->activity_id) {
426 		dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n",
427 			req->dev_id.as_uint32, resp->dev_id.as_uint32,
428 			req->activity_id, resp->activity_id);
429 		return -EPROTO;
430 	}
431 
432 	return 0;
433 }
434 
mana_verify_resp_hdr(const struct gdma_resp_hdr * resp_hdr,const enum mana_command_code expected_code,const u32 min_size)435 static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr,
436 				const enum mana_command_code expected_code,
437 				const u32 min_size)
438 {
439 	if (resp_hdr->response.msg_type != expected_code)
440 		return -EPROTO;
441 
442 	if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1)
443 		return -EPROTO;
444 
445 	if (resp_hdr->response.msg_size < min_size)
446 		return -EPROTO;
447 
448 	return 0;
449 }
450 
mana_pf_register_hw_vport(struct mana_port_context * apc)451 static int mana_pf_register_hw_vport(struct mana_port_context *apc)
452 {
453 	struct mana_register_hw_vport_resp resp = {};
454 	struct mana_register_hw_vport_req req = {};
455 	int err;
456 
457 	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT,
458 			     sizeof(req), sizeof(resp));
459 	req.attached_gfid = 1;
460 	req.is_pf_default_vport = 1;
461 	req.allow_all_ether_types = 1;
462 
463 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
464 				sizeof(resp));
465 	if (err) {
466 		netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err);
467 		return err;
468 	}
469 
470 	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT,
471 				   sizeof(resp));
472 	if (err || resp.hdr.status) {
473 		netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n",
474 			   err, resp.hdr.status);
475 		return err ? err : -EPROTO;
476 	}
477 
478 	apc->port_handle = resp.hw_vport_handle;
479 	return 0;
480 }
481 
mana_pf_deregister_hw_vport(struct mana_port_context * apc)482 static void mana_pf_deregister_hw_vport(struct mana_port_context *apc)
483 {
484 	struct mana_deregister_hw_vport_resp resp = {};
485 	struct mana_deregister_hw_vport_req req = {};
486 	int err;
487 
488 	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT,
489 			     sizeof(req), sizeof(resp));
490 	req.hw_vport_handle = apc->port_handle;
491 
492 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
493 				sizeof(resp));
494 	if (err) {
495 		netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n",
496 			   err);
497 		return;
498 	}
499 
500 	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT,
501 				   sizeof(resp));
502 	if (err || resp.hdr.status)
503 		netdev_err(apc->ndev,
504 			   "Failed to deregister hw vPort: %d, 0x%x\n",
505 			   err, resp.hdr.status);
506 }
507 
mana_pf_register_filter(struct mana_port_context * apc)508 static int mana_pf_register_filter(struct mana_port_context *apc)
509 {
510 	struct mana_register_filter_resp resp = {};
511 	struct mana_register_filter_req req = {};
512 	int err;
513 
514 	mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER,
515 			     sizeof(req), sizeof(resp));
516 	req.vport = apc->port_handle;
517 	memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN);
518 
519 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
520 				sizeof(resp));
521 	if (err) {
522 		netdev_err(apc->ndev, "Failed to register filter: %d\n", err);
523 		return err;
524 	}
525 
526 	err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER,
527 				   sizeof(resp));
528 	if (err || resp.hdr.status) {
529 		netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n",
530 			   err, resp.hdr.status);
531 		return err ? err : -EPROTO;
532 	}
533 
534 	apc->pf_filter_handle = resp.filter_handle;
535 	return 0;
536 }
537 
mana_pf_deregister_filter(struct mana_port_context * apc)538 static void mana_pf_deregister_filter(struct mana_port_context *apc)
539 {
540 	struct mana_deregister_filter_resp resp = {};
541 	struct mana_deregister_filter_req req = {};
542 	int err;
543 
544 	mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER,
545 			     sizeof(req), sizeof(resp));
546 	req.filter_handle = apc->pf_filter_handle;
547 
548 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
549 				sizeof(resp));
550 	if (err) {
551 		netdev_err(apc->ndev, "Failed to unregister filter: %d\n",
552 			   err);
553 		return;
554 	}
555 
556 	err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER,
557 				   sizeof(resp));
558 	if (err || resp.hdr.status)
559 		netdev_err(apc->ndev,
560 			   "Failed to deregister filter: %d, 0x%x\n",
561 			   err, resp.hdr.status);
562 }
563 
mana_query_device_cfg(struct mana_context * ac,u32 proto_major_ver,u32 proto_minor_ver,u32 proto_micro_ver,u16 * max_num_vports)564 static int mana_query_device_cfg(struct mana_context *ac, u32 proto_major_ver,
565 				 u32 proto_minor_ver, u32 proto_micro_ver,
566 				 u16 *max_num_vports)
567 {
568 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
569 	struct mana_query_device_cfg_resp resp = {};
570 	struct mana_query_device_cfg_req req = {};
571 	struct device *dev = gc->dev;
572 	int err = 0;
573 
574 	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG,
575 			     sizeof(req), sizeof(resp));
576 	req.proto_major_ver = proto_major_ver;
577 	req.proto_minor_ver = proto_minor_ver;
578 	req.proto_micro_ver = proto_micro_ver;
579 
580 	err = mana_send_request(ac, &req, sizeof(req), &resp, sizeof(resp));
581 	if (err) {
582 		dev_err(dev, "Failed to query config: %d", err);
583 		return err;
584 	}
585 
586 	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG,
587 				   sizeof(resp));
588 	if (err || resp.hdr.status) {
589 		dev_err(dev, "Invalid query result: %d, 0x%x\n", err,
590 			resp.hdr.status);
591 		if (!err)
592 			err = -EPROTO;
593 		return err;
594 	}
595 
596 	*max_num_vports = resp.max_num_vports;
597 
598 	return 0;
599 }
600 
mana_query_vport_cfg(struct mana_port_context * apc,u32 vport_index,u32 * max_sq,u32 * max_rq,u32 * num_indir_entry)601 static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index,
602 				u32 *max_sq, u32 *max_rq, u32 *num_indir_entry)
603 {
604 	struct mana_query_vport_cfg_resp resp = {};
605 	struct mana_query_vport_cfg_req req = {};
606 	int err;
607 
608 	mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG,
609 			     sizeof(req), sizeof(resp));
610 
611 	req.vport_index = vport_index;
612 
613 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
614 				sizeof(resp));
615 	if (err)
616 		return err;
617 
618 	err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG,
619 				   sizeof(resp));
620 	if (err)
621 		return err;
622 
623 	if (resp.hdr.status)
624 		return -EPROTO;
625 
626 	*max_sq = resp.max_num_sq;
627 	*max_rq = resp.max_num_rq;
628 	*num_indir_entry = resp.num_indirection_ent;
629 
630 	apc->port_handle = resp.vport;
631 	ether_addr_copy(apc->mac_addr, resp.mac_addr);
632 
633 	return 0;
634 }
635 
mana_cfg_vport(struct mana_port_context * apc,u32 protection_dom_id,u32 doorbell_pg_id)636 static int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id,
637 			  u32 doorbell_pg_id)
638 {
639 	struct mana_config_vport_resp resp = {};
640 	struct mana_config_vport_req req = {};
641 	int err;
642 
643 	mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX,
644 			     sizeof(req), sizeof(resp));
645 	req.vport = apc->port_handle;
646 	req.pdid = protection_dom_id;
647 	req.doorbell_pageid = doorbell_pg_id;
648 
649 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
650 				sizeof(resp));
651 	if (err) {
652 		netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err);
653 		goto out;
654 	}
655 
656 	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX,
657 				   sizeof(resp));
658 	if (err || resp.hdr.status) {
659 		netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n",
660 			   err, resp.hdr.status);
661 		if (!err)
662 			err = -EPROTO;
663 
664 		goto out;
665 	}
666 
667 	apc->tx_shortform_allowed = resp.short_form_allowed;
668 	apc->tx_vp_offset = resp.tx_vport_offset;
669 out:
670 	return err;
671 }
672 
mana_cfg_vport_steering(struct mana_port_context * apc,enum TRI_STATE rx,bool update_default_rxobj,bool update_key,bool update_tab)673 static int mana_cfg_vport_steering(struct mana_port_context *apc,
674 				   enum TRI_STATE rx,
675 				   bool update_default_rxobj, bool update_key,
676 				   bool update_tab)
677 {
678 	u16 num_entries = MANA_INDIRECT_TABLE_SIZE;
679 	struct mana_cfg_rx_steer_req *req = NULL;
680 	struct mana_cfg_rx_steer_resp resp = {};
681 	struct net_device *ndev = apc->ndev;
682 	mana_handle_t *req_indir_tab;
683 	u32 req_buf_size;
684 	int err;
685 
686 	req_buf_size = sizeof(*req) + sizeof(mana_handle_t) * num_entries;
687 	req = kzalloc(req_buf_size, GFP_KERNEL);
688 	if (!req)
689 		return -ENOMEM;
690 
691 	mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size,
692 			     sizeof(resp));
693 
694 	req->vport = apc->port_handle;
695 	req->num_indir_entries = num_entries;
696 	req->indir_tab_offset = sizeof(*req);
697 	req->rx_enable = rx;
698 	req->rss_enable = apc->rss_state;
699 	req->update_default_rxobj = update_default_rxobj;
700 	req->update_hashkey = update_key;
701 	req->update_indir_tab = update_tab;
702 	req->default_rxobj = apc->default_rxobj;
703 
704 	if (update_key)
705 		memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE);
706 
707 	if (update_tab) {
708 		req_indir_tab = (mana_handle_t *)(req + 1);
709 		memcpy(req_indir_tab, apc->rxobj_table,
710 		       req->num_indir_entries * sizeof(mana_handle_t));
711 	}
712 
713 	err = mana_send_request(apc->ac, req, req_buf_size, &resp,
714 				sizeof(resp));
715 	if (err) {
716 		netdev_err(ndev, "Failed to configure vPort RX: %d\n", err);
717 		goto out;
718 	}
719 
720 	err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX,
721 				   sizeof(resp));
722 	if (err) {
723 		netdev_err(ndev, "vPort RX configuration failed: %d\n", err);
724 		goto out;
725 	}
726 
727 	if (resp.hdr.status) {
728 		netdev_err(ndev, "vPort RX configuration failed: 0x%x\n",
729 			   resp.hdr.status);
730 		err = -EPROTO;
731 	}
732 out:
733 	kfree(req);
734 	return err;
735 }
736 
mana_create_wq_obj(struct mana_port_context * apc,mana_handle_t vport,u32 wq_type,struct mana_obj_spec * wq_spec,struct mana_obj_spec * cq_spec,mana_handle_t * wq_obj)737 static int mana_create_wq_obj(struct mana_port_context *apc,
738 			      mana_handle_t vport,
739 			      u32 wq_type, struct mana_obj_spec *wq_spec,
740 			      struct mana_obj_spec *cq_spec,
741 			      mana_handle_t *wq_obj)
742 {
743 	struct mana_create_wqobj_resp resp = {};
744 	struct mana_create_wqobj_req req = {};
745 	struct net_device *ndev = apc->ndev;
746 	int err;
747 
748 	mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ,
749 			     sizeof(req), sizeof(resp));
750 	req.vport = vport;
751 	req.wq_type = wq_type;
752 	req.wq_gdma_region = wq_spec->gdma_region;
753 	req.cq_gdma_region = cq_spec->gdma_region;
754 	req.wq_size = wq_spec->queue_size;
755 	req.cq_size = cq_spec->queue_size;
756 	req.cq_moderation_ctx_id = cq_spec->modr_ctx_id;
757 	req.cq_parent_qid = cq_spec->attached_eq;
758 
759 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
760 				sizeof(resp));
761 	if (err) {
762 		netdev_err(ndev, "Failed to create WQ object: %d\n", err);
763 		goto out;
764 	}
765 
766 	err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ,
767 				   sizeof(resp));
768 	if (err || resp.hdr.status) {
769 		netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err,
770 			   resp.hdr.status);
771 		if (!err)
772 			err = -EPROTO;
773 		goto out;
774 	}
775 
776 	if (resp.wq_obj == INVALID_MANA_HANDLE) {
777 		netdev_err(ndev, "Got an invalid WQ object handle\n");
778 		err = -EPROTO;
779 		goto out;
780 	}
781 
782 	*wq_obj = resp.wq_obj;
783 	wq_spec->queue_index = resp.wq_id;
784 	cq_spec->queue_index = resp.cq_id;
785 
786 	return 0;
787 out:
788 	return err;
789 }
790 
mana_destroy_wq_obj(struct mana_port_context * apc,u32 wq_type,mana_handle_t wq_obj)791 static void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type,
792 				mana_handle_t wq_obj)
793 {
794 	struct mana_destroy_wqobj_resp resp = {};
795 	struct mana_destroy_wqobj_req req = {};
796 	struct net_device *ndev = apc->ndev;
797 	int err;
798 
799 	mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ,
800 			     sizeof(req), sizeof(resp));
801 	req.wq_type = wq_type;
802 	req.wq_obj_handle = wq_obj;
803 
804 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
805 				sizeof(resp));
806 	if (err) {
807 		netdev_err(ndev, "Failed to destroy WQ object: %d\n", err);
808 		return;
809 	}
810 
811 	err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ,
812 				   sizeof(resp));
813 	if (err || resp.hdr.status)
814 		netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err,
815 			   resp.hdr.status);
816 }
817 
mana_destroy_eq(struct mana_context * ac)818 static void mana_destroy_eq(struct mana_context *ac)
819 {
820 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
821 	struct gdma_queue *eq;
822 	int i;
823 
824 	if (!ac->eqs)
825 		return;
826 
827 	for (i = 0; i < gc->max_num_queues; i++) {
828 		eq = ac->eqs[i].eq;
829 		if (!eq)
830 			continue;
831 
832 		mana_gd_destroy_queue(gc, eq);
833 	}
834 
835 	kfree(ac->eqs);
836 	ac->eqs = NULL;
837 }
838 
mana_create_eq(struct mana_context * ac)839 static int mana_create_eq(struct mana_context *ac)
840 {
841 	struct gdma_dev *gd = ac->gdma_dev;
842 	struct gdma_context *gc = gd->gdma_context;
843 	struct gdma_queue_spec spec = {};
844 	int err;
845 	int i;
846 
847 	ac->eqs = kcalloc(gc->max_num_queues, sizeof(struct mana_eq),
848 			  GFP_KERNEL);
849 	if (!ac->eqs)
850 		return -ENOMEM;
851 
852 	spec.type = GDMA_EQ;
853 	spec.monitor_avl_buf = false;
854 	spec.queue_size = EQ_SIZE;
855 	spec.eq.callback = NULL;
856 	spec.eq.context = ac->eqs;
857 	spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE;
858 
859 	for (i = 0; i < gc->max_num_queues; i++) {
860 		err = mana_gd_create_mana_eq(gd, &spec, &ac->eqs[i].eq);
861 		if (err)
862 			goto out;
863 	}
864 
865 	return 0;
866 out:
867 	mana_destroy_eq(ac);
868 	return err;
869 }
870 
mana_fence_rq(struct mana_port_context * apc,struct mana_rxq * rxq)871 static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq)
872 {
873 	struct mana_fence_rq_resp resp = {};
874 	struct mana_fence_rq_req req = {};
875 	int err;
876 
877 	init_completion(&rxq->fence_event);
878 
879 	mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ,
880 			     sizeof(req), sizeof(resp));
881 	req.wq_obj_handle =  rxq->rxobj;
882 
883 	err = mana_send_request(apc->ac, &req, sizeof(req), &resp,
884 				sizeof(resp));
885 	if (err) {
886 		netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n",
887 			   rxq->rxq_idx, err);
888 		return err;
889 	}
890 
891 	err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp));
892 	if (err || resp.hdr.status) {
893 		netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n",
894 			   rxq->rxq_idx, err, resp.hdr.status);
895 		if (!err)
896 			err = -EPROTO;
897 
898 		return err;
899 	}
900 
901 	if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) {
902 		netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n",
903 			   rxq->rxq_idx);
904 		return -ETIMEDOUT;
905 	}
906 
907 	return 0;
908 }
909 
mana_fence_rqs(struct mana_port_context * apc)910 static void mana_fence_rqs(struct mana_port_context *apc)
911 {
912 	unsigned int rxq_idx;
913 	struct mana_rxq *rxq;
914 	int err;
915 
916 	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
917 		rxq = apc->rxqs[rxq_idx];
918 		err = mana_fence_rq(apc, rxq);
919 
920 		/* In case of any error, use sleep instead. */
921 		if (err)
922 			msleep(100);
923 	}
924 }
925 
mana_move_wq_tail(struct gdma_queue * wq,u32 num_units)926 static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units)
927 {
928 	u32 used_space_old;
929 	u32 used_space_new;
930 
931 	used_space_old = wq->head - wq->tail;
932 	used_space_new = wq->head - (wq->tail + num_units);
933 
934 	if (WARN_ON_ONCE(used_space_new > used_space_old))
935 		return -ERANGE;
936 
937 	wq->tail += num_units;
938 	return 0;
939 }
940 
mana_unmap_skb(struct sk_buff * skb,struct mana_port_context * apc)941 static void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc)
942 {
943 	struct mana_skb_head *ash = (struct mana_skb_head *)skb->head;
944 	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
945 	struct device *dev = gc->dev;
946 	int i;
947 
948 	dma_unmap_single(dev, ash->dma_handle[0], ash->size[0], DMA_TO_DEVICE);
949 
950 	for (i = 1; i < skb_shinfo(skb)->nr_frags + 1; i++)
951 		dma_unmap_page(dev, ash->dma_handle[i], ash->size[i],
952 			       DMA_TO_DEVICE);
953 }
954 
mana_poll_tx_cq(struct mana_cq * cq)955 static void mana_poll_tx_cq(struct mana_cq *cq)
956 {
957 	struct gdma_comp *completions = cq->gdma_comp_buf;
958 	struct gdma_posted_wqe_info *wqe_info;
959 	unsigned int pkt_transmitted = 0;
960 	unsigned int wqe_unit_cnt = 0;
961 	struct mana_txq *txq = cq->txq;
962 	struct mana_port_context *apc;
963 	struct netdev_queue *net_txq;
964 	struct gdma_queue *gdma_wq;
965 	unsigned int avail_space;
966 	struct net_device *ndev;
967 	struct sk_buff *skb;
968 	bool txq_stopped;
969 	int comp_read;
970 	int i;
971 
972 	ndev = txq->ndev;
973 	apc = netdev_priv(ndev);
974 
975 	comp_read = mana_gd_poll_cq(cq->gdma_cq, completions,
976 				    CQE_POLLING_BUFFER);
977 
978 	if (comp_read < 1)
979 		return;
980 
981 	for (i = 0; i < comp_read; i++) {
982 		struct mana_tx_comp_oob *cqe_oob;
983 
984 		if (WARN_ON_ONCE(!completions[i].is_sq))
985 			return;
986 
987 		cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data;
988 		if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type !=
989 				 MANA_CQE_COMPLETION))
990 			return;
991 
992 		switch (cqe_oob->cqe_hdr.cqe_type) {
993 		case CQE_TX_OKAY:
994 			break;
995 
996 		case CQE_TX_SA_DROP:
997 		case CQE_TX_MTU_DROP:
998 		case CQE_TX_INVALID_OOB:
999 		case CQE_TX_INVALID_ETH_TYPE:
1000 		case CQE_TX_HDR_PROCESSING_ERROR:
1001 		case CQE_TX_VF_DISABLED:
1002 		case CQE_TX_VPORT_IDX_OUT_OF_RANGE:
1003 		case CQE_TX_VPORT_DISABLED:
1004 		case CQE_TX_VLAN_TAGGING_VIOLATION:
1005 			WARN_ONCE(1, "TX: CQE error %d: ignored.\n",
1006 				  cqe_oob->cqe_hdr.cqe_type);
1007 			break;
1008 
1009 		default:
1010 			/* If the CQE type is unexpected, log an error, assert,
1011 			 * and go through the error path.
1012 			 */
1013 			WARN_ONCE(1, "TX: Unexpected CQE type %d: HW BUG?\n",
1014 				  cqe_oob->cqe_hdr.cqe_type);
1015 			return;
1016 		}
1017 
1018 		if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num))
1019 			return;
1020 
1021 		skb = skb_dequeue(&txq->pending_skbs);
1022 		if (WARN_ON_ONCE(!skb))
1023 			return;
1024 
1025 		wqe_info = (struct gdma_posted_wqe_info *)skb->cb;
1026 		wqe_unit_cnt += wqe_info->wqe_size_in_bu;
1027 
1028 		mana_unmap_skb(skb, apc);
1029 
1030 		napi_consume_skb(skb, cq->budget);
1031 
1032 		pkt_transmitted++;
1033 	}
1034 
1035 	if (WARN_ON_ONCE(wqe_unit_cnt == 0))
1036 		return;
1037 
1038 	mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt);
1039 
1040 	gdma_wq = txq->gdma_sq;
1041 	avail_space = mana_gd_wq_avail_space(gdma_wq);
1042 
1043 	/* Ensure tail updated before checking q stop */
1044 	smp_mb();
1045 
1046 	net_txq = txq->net_txq;
1047 	txq_stopped = netif_tx_queue_stopped(net_txq);
1048 
1049 	/* Ensure checking txq_stopped before apc->port_is_up. */
1050 	smp_rmb();
1051 
1052 	if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) {
1053 		netif_tx_wake_queue(net_txq);
1054 		apc->eth_stats.wake_queue++;
1055 	}
1056 
1057 	if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0)
1058 		WARN_ON_ONCE(1);
1059 
1060 	cq->work_done = pkt_transmitted;
1061 }
1062 
mana_post_pkt_rxq(struct mana_rxq * rxq)1063 static void mana_post_pkt_rxq(struct mana_rxq *rxq)
1064 {
1065 	struct mana_recv_buf_oob *recv_buf_oob;
1066 	u32 curr_index;
1067 	int err;
1068 
1069 	curr_index = rxq->buf_index++;
1070 	if (rxq->buf_index == rxq->num_rx_buf)
1071 		rxq->buf_index = 0;
1072 
1073 	recv_buf_oob = &rxq->rx_oobs[curr_index];
1074 
1075 	err = mana_gd_post_and_ring(rxq->gdma_rq, &recv_buf_oob->wqe_req,
1076 				    &recv_buf_oob->wqe_inf);
1077 	if (WARN_ON_ONCE(err))
1078 		return;
1079 
1080 	WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1);
1081 }
1082 
mana_build_skb(void * buf_va,uint pkt_len,struct xdp_buff * xdp)1083 static struct sk_buff *mana_build_skb(void *buf_va, uint pkt_len,
1084 				      struct xdp_buff *xdp)
1085 {
1086 	struct sk_buff *skb = build_skb(buf_va, PAGE_SIZE);
1087 
1088 	if (!skb)
1089 		return NULL;
1090 
1091 	if (xdp->data_hard_start) {
1092 		skb_reserve(skb, xdp->data - xdp->data_hard_start);
1093 		skb_put(skb, xdp->data_end - xdp->data);
1094 	} else {
1095 		skb_reserve(skb, XDP_PACKET_HEADROOM);
1096 		skb_put(skb, pkt_len);
1097 	}
1098 
1099 	return skb;
1100 }
1101 
mana_rx_skb(void * buf_va,struct mana_rxcomp_oob * cqe,struct mana_rxq * rxq)1102 static void mana_rx_skb(void *buf_va, struct mana_rxcomp_oob *cqe,
1103 			struct mana_rxq *rxq)
1104 {
1105 	struct mana_stats_rx *rx_stats = &rxq->stats;
1106 	struct net_device *ndev = rxq->ndev;
1107 	uint pkt_len = cqe->ppi[0].pkt_len;
1108 	u16 rxq_idx = rxq->rxq_idx;
1109 	struct napi_struct *napi;
1110 	struct xdp_buff xdp = {};
1111 	struct sk_buff *skb;
1112 	u32 hash_value;
1113 	u32 act;
1114 
1115 	rxq->rx_cq.work_done++;
1116 	napi = &rxq->rx_cq.napi;
1117 
1118 	if (!buf_va) {
1119 		++ndev->stats.rx_dropped;
1120 		return;
1121 	}
1122 
1123 	act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len);
1124 
1125 	if (act == XDP_REDIRECT && !rxq->xdp_rc)
1126 		return;
1127 
1128 	if (act != XDP_PASS && act != XDP_TX)
1129 		goto drop_xdp;
1130 
1131 	skb = mana_build_skb(buf_va, pkt_len, &xdp);
1132 
1133 	if (!skb)
1134 		goto drop;
1135 
1136 	skb->dev = napi->dev;
1137 
1138 	skb->protocol = eth_type_trans(skb, ndev);
1139 	skb_checksum_none_assert(skb);
1140 	skb_record_rx_queue(skb, rxq_idx);
1141 
1142 	if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) {
1143 		if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed)
1144 			skb->ip_summed = CHECKSUM_UNNECESSARY;
1145 	}
1146 
1147 	if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) {
1148 		hash_value = cqe->ppi[0].pkt_hash;
1149 
1150 		if (cqe->rx_hashtype & MANA_HASH_L4)
1151 			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L4);
1152 		else
1153 			skb_set_hash(skb, hash_value, PKT_HASH_TYPE_L3);
1154 	}
1155 
1156 	u64_stats_update_begin(&rx_stats->syncp);
1157 	rx_stats->packets++;
1158 	rx_stats->bytes += pkt_len;
1159 
1160 	if (act == XDP_TX)
1161 		rx_stats->xdp_tx++;
1162 	u64_stats_update_end(&rx_stats->syncp);
1163 
1164 	if (act == XDP_TX) {
1165 		skb_set_queue_mapping(skb, rxq_idx);
1166 		mana_xdp_tx(skb, ndev);
1167 		return;
1168 	}
1169 
1170 	napi_gro_receive(napi, skb);
1171 
1172 	return;
1173 
1174 drop_xdp:
1175 	u64_stats_update_begin(&rx_stats->syncp);
1176 	rx_stats->xdp_drop++;
1177 	u64_stats_update_end(&rx_stats->syncp);
1178 
1179 drop:
1180 	WARN_ON_ONCE(rxq->xdp_save_page);
1181 	rxq->xdp_save_page = virt_to_page(buf_va);
1182 
1183 	++ndev->stats.rx_dropped;
1184 
1185 	return;
1186 }
1187 
mana_process_rx_cqe(struct mana_rxq * rxq,struct mana_cq * cq,struct gdma_comp * cqe)1188 static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq,
1189 				struct gdma_comp *cqe)
1190 {
1191 	struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data;
1192 	struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context;
1193 	struct net_device *ndev = rxq->ndev;
1194 	struct mana_recv_buf_oob *rxbuf_oob;
1195 	struct device *dev = gc->dev;
1196 	void *new_buf, *old_buf;
1197 	struct page *new_page;
1198 	u32 curr, pktlen;
1199 	dma_addr_t da;
1200 
1201 	switch (oob->cqe_hdr.cqe_type) {
1202 	case CQE_RX_OKAY:
1203 		break;
1204 
1205 	case CQE_RX_TRUNCATED:
1206 		++ndev->stats.rx_dropped;
1207 		rxbuf_oob = &rxq->rx_oobs[rxq->buf_index];
1208 		netdev_warn_once(ndev, "Dropped a truncated packet\n");
1209 		goto drop;
1210 
1211 	case CQE_RX_COALESCED_4:
1212 		netdev_err(ndev, "RX coalescing is unsupported\n");
1213 		return;
1214 
1215 	case CQE_RX_OBJECT_FENCE:
1216 		complete(&rxq->fence_event);
1217 		return;
1218 
1219 	default:
1220 		netdev_err(ndev, "Unknown RX CQE type = %d\n",
1221 			   oob->cqe_hdr.cqe_type);
1222 		return;
1223 	}
1224 
1225 	pktlen = oob->ppi[0].pkt_len;
1226 
1227 	if (pktlen == 0) {
1228 		/* data packets should never have packetlength of zero */
1229 		netdev_err(ndev, "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n",
1230 			   rxq->gdma_id, cq->gdma_id, rxq->rxobj);
1231 		return;
1232 	}
1233 
1234 	curr = rxq->buf_index;
1235 	rxbuf_oob = &rxq->rx_oobs[curr];
1236 	WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1);
1237 
1238 	/* Reuse XDP dropped page if available */
1239 	if (rxq->xdp_save_page) {
1240 		new_page = rxq->xdp_save_page;
1241 		rxq->xdp_save_page = NULL;
1242 	} else {
1243 		new_page = alloc_page(GFP_ATOMIC);
1244 	}
1245 
1246 	if (new_page) {
1247 		da = dma_map_page(dev, new_page, XDP_PACKET_HEADROOM, rxq->datasize,
1248 				  DMA_FROM_DEVICE);
1249 
1250 		if (dma_mapping_error(dev, da)) {
1251 			__free_page(new_page);
1252 			new_page = NULL;
1253 		}
1254 	}
1255 
1256 	new_buf = new_page ? page_to_virt(new_page) : NULL;
1257 
1258 	if (new_buf) {
1259 		dma_unmap_page(dev, rxbuf_oob->buf_dma_addr, rxq->datasize,
1260 			       DMA_FROM_DEVICE);
1261 
1262 		old_buf = rxbuf_oob->buf_va;
1263 
1264 		/* refresh the rxbuf_oob with the new page */
1265 		rxbuf_oob->buf_va = new_buf;
1266 		rxbuf_oob->buf_dma_addr = da;
1267 		rxbuf_oob->sgl[0].address = rxbuf_oob->buf_dma_addr;
1268 	} else {
1269 		old_buf = NULL; /* drop the packet if no memory */
1270 	}
1271 
1272 	mana_rx_skb(old_buf, oob, rxq);
1273 
1274 drop:
1275 	mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu);
1276 
1277 	mana_post_pkt_rxq(rxq);
1278 }
1279 
mana_poll_rx_cq(struct mana_cq * cq)1280 static void mana_poll_rx_cq(struct mana_cq *cq)
1281 {
1282 	struct gdma_comp *comp = cq->gdma_comp_buf;
1283 	struct mana_rxq *rxq = cq->rxq;
1284 	int comp_read, i;
1285 
1286 	comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, CQE_POLLING_BUFFER);
1287 	WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER);
1288 
1289 	rxq->xdp_flush = false;
1290 
1291 	for (i = 0; i < comp_read; i++) {
1292 		if (WARN_ON_ONCE(comp[i].is_sq))
1293 			return;
1294 
1295 		/* verify recv cqe references the right rxq */
1296 		if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id))
1297 			return;
1298 
1299 		mana_process_rx_cqe(rxq, cq, &comp[i]);
1300 	}
1301 
1302 	if (rxq->xdp_flush)
1303 		xdp_do_flush();
1304 }
1305 
mana_cq_handler(void * context,struct gdma_queue * gdma_queue)1306 static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue)
1307 {
1308 	struct mana_cq *cq = context;
1309 	u8 arm_bit;
1310 	int w;
1311 
1312 	WARN_ON_ONCE(cq->gdma_cq != gdma_queue);
1313 
1314 	if (cq->type == MANA_CQ_TYPE_RX)
1315 		mana_poll_rx_cq(cq);
1316 	else
1317 		mana_poll_tx_cq(cq);
1318 
1319 	w = cq->work_done;
1320 
1321 	if (w < cq->budget &&
1322 	    napi_complete_done(&cq->napi, w)) {
1323 		arm_bit = SET_ARM_BIT;
1324 	} else {
1325 		arm_bit = 0;
1326 	}
1327 
1328 	mana_gd_ring_cq(gdma_queue, arm_bit);
1329 
1330 	return w;
1331 }
1332 
mana_poll(struct napi_struct * napi,int budget)1333 static int mana_poll(struct napi_struct *napi, int budget)
1334 {
1335 	struct mana_cq *cq = container_of(napi, struct mana_cq, napi);
1336 	int w;
1337 
1338 	cq->work_done = 0;
1339 	cq->budget = budget;
1340 
1341 	w = mana_cq_handler(cq, cq->gdma_cq);
1342 
1343 	return min(w, budget);
1344 }
1345 
mana_schedule_napi(void * context,struct gdma_queue * gdma_queue)1346 static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue)
1347 {
1348 	struct mana_cq *cq = context;
1349 
1350 	napi_schedule_irqoff(&cq->napi);
1351 }
1352 
mana_deinit_cq(struct mana_port_context * apc,struct mana_cq * cq)1353 static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq)
1354 {
1355 	struct gdma_dev *gd = apc->ac->gdma_dev;
1356 
1357 	if (!cq->gdma_cq)
1358 		return;
1359 
1360 	mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq);
1361 }
1362 
mana_deinit_txq(struct mana_port_context * apc,struct mana_txq * txq)1363 static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq)
1364 {
1365 	struct gdma_dev *gd = apc->ac->gdma_dev;
1366 
1367 	if (!txq->gdma_sq)
1368 		return;
1369 
1370 	mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq);
1371 }
1372 
mana_destroy_txq(struct mana_port_context * apc)1373 static void mana_destroy_txq(struct mana_port_context *apc)
1374 {
1375 	struct napi_struct *napi;
1376 	int i;
1377 
1378 	if (!apc->tx_qp)
1379 		return;
1380 
1381 	for (i = 0; i < apc->num_queues; i++) {
1382 		napi = &apc->tx_qp[i].tx_cq.napi;
1383 		napi_synchronize(napi);
1384 		napi_disable(napi);
1385 		netif_napi_del(napi);
1386 
1387 		mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i].tx_object);
1388 
1389 		mana_deinit_cq(apc, &apc->tx_qp[i].tx_cq);
1390 
1391 		mana_deinit_txq(apc, &apc->tx_qp[i].txq);
1392 	}
1393 
1394 	kfree(apc->tx_qp);
1395 	apc->tx_qp = NULL;
1396 }
1397 
mana_create_txq(struct mana_port_context * apc,struct net_device * net)1398 static int mana_create_txq(struct mana_port_context *apc,
1399 			   struct net_device *net)
1400 {
1401 	struct mana_context *ac = apc->ac;
1402 	struct gdma_dev *gd = ac->gdma_dev;
1403 	struct mana_obj_spec wq_spec;
1404 	struct mana_obj_spec cq_spec;
1405 	struct gdma_queue_spec spec;
1406 	struct gdma_context *gc;
1407 	struct mana_txq *txq;
1408 	struct mana_cq *cq;
1409 	u32 txq_size;
1410 	u32 cq_size;
1411 	int err;
1412 	int i;
1413 
1414 	apc->tx_qp = kcalloc(apc->num_queues, sizeof(struct mana_tx_qp),
1415 			     GFP_KERNEL);
1416 	if (!apc->tx_qp)
1417 		return -ENOMEM;
1418 
1419 	/*  The minimum size of the WQE is 32 bytes, hence
1420 	 *  MAX_SEND_BUFFERS_PER_QUEUE represents the maximum number of WQEs
1421 	 *  the SQ can store. This value is then used to size other queues
1422 	 *  to prevent overflow.
1423 	 */
1424 	txq_size = MAX_SEND_BUFFERS_PER_QUEUE * 32;
1425 	BUILD_BUG_ON(!PAGE_ALIGNED(txq_size));
1426 
1427 	cq_size = MAX_SEND_BUFFERS_PER_QUEUE * COMP_ENTRY_SIZE;
1428 	cq_size = PAGE_ALIGN(cq_size);
1429 
1430 	gc = gd->gdma_context;
1431 
1432 	for (i = 0; i < apc->num_queues; i++) {
1433 		apc->tx_qp[i].tx_object = INVALID_MANA_HANDLE;
1434 
1435 		/* Create SQ */
1436 		txq = &apc->tx_qp[i].txq;
1437 
1438 		u64_stats_init(&txq->stats.syncp);
1439 		txq->ndev = net;
1440 		txq->net_txq = netdev_get_tx_queue(net, i);
1441 		txq->vp_offset = apc->tx_vp_offset;
1442 		skb_queue_head_init(&txq->pending_skbs);
1443 
1444 		memset(&spec, 0, sizeof(spec));
1445 		spec.type = GDMA_SQ;
1446 		spec.monitor_avl_buf = true;
1447 		spec.queue_size = txq_size;
1448 		err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq);
1449 		if (err)
1450 			goto out;
1451 
1452 		/* Create SQ's CQ */
1453 		cq = &apc->tx_qp[i].tx_cq;
1454 		cq->type = MANA_CQ_TYPE_TX;
1455 
1456 		cq->txq = txq;
1457 
1458 		memset(&spec, 0, sizeof(spec));
1459 		spec.type = GDMA_CQ;
1460 		spec.monitor_avl_buf = false;
1461 		spec.queue_size = cq_size;
1462 		spec.cq.callback = mana_schedule_napi;
1463 		spec.cq.parent_eq = ac->eqs[i].eq;
1464 		spec.cq.context = cq;
1465 		err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
1466 		if (err)
1467 			goto out;
1468 
1469 		memset(&wq_spec, 0, sizeof(wq_spec));
1470 		memset(&cq_spec, 0, sizeof(cq_spec));
1471 
1472 		wq_spec.gdma_region = txq->gdma_sq->mem_info.gdma_region;
1473 		wq_spec.queue_size = txq->gdma_sq->queue_size;
1474 
1475 		cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region;
1476 		cq_spec.queue_size = cq->gdma_cq->queue_size;
1477 		cq_spec.modr_ctx_id = 0;
1478 		cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
1479 
1480 		err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ,
1481 					 &wq_spec, &cq_spec,
1482 					 &apc->tx_qp[i].tx_object);
1483 
1484 		if (err)
1485 			goto out;
1486 
1487 		txq->gdma_sq->id = wq_spec.queue_index;
1488 		cq->gdma_cq->id = cq_spec.queue_index;
1489 
1490 		txq->gdma_sq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1491 		cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1492 
1493 		txq->gdma_txq_id = txq->gdma_sq->id;
1494 
1495 		cq->gdma_id = cq->gdma_cq->id;
1496 
1497 		if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
1498 			err = -EINVAL;
1499 			goto out;
1500 		}
1501 
1502 		gc->cq_table[cq->gdma_id] = cq->gdma_cq;
1503 
1504 		netif_napi_add_tx(net, &cq->napi, mana_poll);
1505 		napi_enable(&cq->napi);
1506 
1507 		mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
1508 	}
1509 
1510 	return 0;
1511 out:
1512 	mana_destroy_txq(apc);
1513 	return err;
1514 }
1515 
mana_destroy_rxq(struct mana_port_context * apc,struct mana_rxq * rxq,bool validate_state)1516 static void mana_destroy_rxq(struct mana_port_context *apc,
1517 			     struct mana_rxq *rxq, bool validate_state)
1518 
1519 {
1520 	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1521 	struct mana_recv_buf_oob *rx_oob;
1522 	struct device *dev = gc->dev;
1523 	struct napi_struct *napi;
1524 	int i;
1525 
1526 	if (!rxq)
1527 		return;
1528 
1529 	napi = &rxq->rx_cq.napi;
1530 
1531 	if (validate_state)
1532 		napi_synchronize(napi);
1533 
1534 	napi_disable(napi);
1535 
1536 	xdp_rxq_info_unreg(&rxq->xdp_rxq);
1537 
1538 	netif_napi_del(napi);
1539 
1540 	mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj);
1541 
1542 	mana_deinit_cq(apc, &rxq->rx_cq);
1543 
1544 	if (rxq->xdp_save_page)
1545 		__free_page(rxq->xdp_save_page);
1546 
1547 	for (i = 0; i < rxq->num_rx_buf; i++) {
1548 		rx_oob = &rxq->rx_oobs[i];
1549 
1550 		if (!rx_oob->buf_va)
1551 			continue;
1552 
1553 		dma_unmap_page(dev, rx_oob->buf_dma_addr, rxq->datasize,
1554 			       DMA_FROM_DEVICE);
1555 
1556 		free_page((unsigned long)rx_oob->buf_va);
1557 		rx_oob->buf_va = NULL;
1558 	}
1559 
1560 	if (rxq->gdma_rq)
1561 		mana_gd_destroy_queue(gc, rxq->gdma_rq);
1562 
1563 	kfree(rxq);
1564 }
1565 
1566 #define MANA_WQE_HEADER_SIZE 16
1567 #define MANA_WQE_SGE_SIZE 16
1568 
mana_alloc_rx_wqe(struct mana_port_context * apc,struct mana_rxq * rxq,u32 * rxq_size,u32 * cq_size)1569 static int mana_alloc_rx_wqe(struct mana_port_context *apc,
1570 			     struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size)
1571 {
1572 	struct gdma_context *gc = apc->ac->gdma_dev->gdma_context;
1573 	struct mana_recv_buf_oob *rx_oob;
1574 	struct device *dev = gc->dev;
1575 	struct page *page;
1576 	dma_addr_t da;
1577 	u32 buf_idx;
1578 
1579 	WARN_ON(rxq->datasize == 0 || rxq->datasize > PAGE_SIZE);
1580 
1581 	*rxq_size = 0;
1582 	*cq_size = 0;
1583 
1584 	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
1585 		rx_oob = &rxq->rx_oobs[buf_idx];
1586 		memset(rx_oob, 0, sizeof(*rx_oob));
1587 
1588 		page = alloc_page(GFP_KERNEL);
1589 		if (!page)
1590 			return -ENOMEM;
1591 
1592 		da = dma_map_page(dev, page, XDP_PACKET_HEADROOM, rxq->datasize,
1593 				  DMA_FROM_DEVICE);
1594 
1595 		if (dma_mapping_error(dev, da)) {
1596 			__free_page(page);
1597 			return -ENOMEM;
1598 		}
1599 
1600 		rx_oob->buf_va = page_to_virt(page);
1601 		rx_oob->buf_dma_addr = da;
1602 
1603 		rx_oob->num_sge = 1;
1604 		rx_oob->sgl[0].address = rx_oob->buf_dma_addr;
1605 		rx_oob->sgl[0].size = rxq->datasize;
1606 		rx_oob->sgl[0].mem_key = apc->ac->gdma_dev->gpa_mkey;
1607 
1608 		rx_oob->wqe_req.sgl = rx_oob->sgl;
1609 		rx_oob->wqe_req.num_sge = rx_oob->num_sge;
1610 		rx_oob->wqe_req.inline_oob_size = 0;
1611 		rx_oob->wqe_req.inline_oob_data = NULL;
1612 		rx_oob->wqe_req.flags = 0;
1613 		rx_oob->wqe_req.client_data_unit = 0;
1614 
1615 		*rxq_size += ALIGN(MANA_WQE_HEADER_SIZE +
1616 				   MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32);
1617 		*cq_size += COMP_ENTRY_SIZE;
1618 	}
1619 
1620 	return 0;
1621 }
1622 
mana_push_wqe(struct mana_rxq * rxq)1623 static int mana_push_wqe(struct mana_rxq *rxq)
1624 {
1625 	struct mana_recv_buf_oob *rx_oob;
1626 	u32 buf_idx;
1627 	int err;
1628 
1629 	for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) {
1630 		rx_oob = &rxq->rx_oobs[buf_idx];
1631 
1632 		err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req,
1633 					    &rx_oob->wqe_inf);
1634 		if (err)
1635 			return -ENOSPC;
1636 	}
1637 
1638 	return 0;
1639 }
1640 
mana_create_rxq(struct mana_port_context * apc,u32 rxq_idx,struct mana_eq * eq,struct net_device * ndev)1641 static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc,
1642 					u32 rxq_idx, struct mana_eq *eq,
1643 					struct net_device *ndev)
1644 {
1645 	struct gdma_dev *gd = apc->ac->gdma_dev;
1646 	struct mana_obj_spec wq_spec;
1647 	struct mana_obj_spec cq_spec;
1648 	struct gdma_queue_spec spec;
1649 	struct mana_cq *cq = NULL;
1650 	struct gdma_context *gc;
1651 	u32 cq_size, rq_size;
1652 	struct mana_rxq *rxq;
1653 	int err;
1654 
1655 	gc = gd->gdma_context;
1656 
1657 	rxq = kzalloc(struct_size(rxq, rx_oobs, RX_BUFFERS_PER_QUEUE),
1658 		      GFP_KERNEL);
1659 	if (!rxq)
1660 		return NULL;
1661 
1662 	rxq->ndev = ndev;
1663 	rxq->num_rx_buf = RX_BUFFERS_PER_QUEUE;
1664 	rxq->rxq_idx = rxq_idx;
1665 	rxq->datasize = ALIGN(MAX_FRAME_SIZE, 64);
1666 	rxq->rxobj = INVALID_MANA_HANDLE;
1667 
1668 	err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size);
1669 	if (err)
1670 		goto out;
1671 
1672 	rq_size = PAGE_ALIGN(rq_size);
1673 	cq_size = PAGE_ALIGN(cq_size);
1674 
1675 	/* Create RQ */
1676 	memset(&spec, 0, sizeof(spec));
1677 	spec.type = GDMA_RQ;
1678 	spec.monitor_avl_buf = true;
1679 	spec.queue_size = rq_size;
1680 	err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq);
1681 	if (err)
1682 		goto out;
1683 
1684 	/* Create RQ's CQ */
1685 	cq = &rxq->rx_cq;
1686 	cq->type = MANA_CQ_TYPE_RX;
1687 	cq->rxq = rxq;
1688 
1689 	memset(&spec, 0, sizeof(spec));
1690 	spec.type = GDMA_CQ;
1691 	spec.monitor_avl_buf = false;
1692 	spec.queue_size = cq_size;
1693 	spec.cq.callback = mana_schedule_napi;
1694 	spec.cq.parent_eq = eq->eq;
1695 	spec.cq.context = cq;
1696 	err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq);
1697 	if (err)
1698 		goto out;
1699 
1700 	memset(&wq_spec, 0, sizeof(wq_spec));
1701 	memset(&cq_spec, 0, sizeof(cq_spec));
1702 	wq_spec.gdma_region = rxq->gdma_rq->mem_info.gdma_region;
1703 	wq_spec.queue_size = rxq->gdma_rq->queue_size;
1704 
1705 	cq_spec.gdma_region = cq->gdma_cq->mem_info.gdma_region;
1706 	cq_spec.queue_size = cq->gdma_cq->queue_size;
1707 	cq_spec.modr_ctx_id = 0;
1708 	cq_spec.attached_eq = cq->gdma_cq->cq.parent->id;
1709 
1710 	err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ,
1711 				 &wq_spec, &cq_spec, &rxq->rxobj);
1712 	if (err)
1713 		goto out;
1714 
1715 	rxq->gdma_rq->id = wq_spec.queue_index;
1716 	cq->gdma_cq->id = cq_spec.queue_index;
1717 
1718 	rxq->gdma_rq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1719 	cq->gdma_cq->mem_info.gdma_region = GDMA_INVALID_DMA_REGION;
1720 
1721 	rxq->gdma_id = rxq->gdma_rq->id;
1722 	cq->gdma_id = cq->gdma_cq->id;
1723 
1724 	err = mana_push_wqe(rxq);
1725 	if (err)
1726 		goto out;
1727 
1728 	if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) {
1729 		err = -EINVAL;
1730 		goto out;
1731 	}
1732 
1733 	gc->cq_table[cq->gdma_id] = cq->gdma_cq;
1734 
1735 	netif_napi_add_weight(ndev, &cq->napi, mana_poll, 1);
1736 
1737 	WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx,
1738 				 cq->napi.napi_id));
1739 	WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq,
1740 					   MEM_TYPE_PAGE_SHARED, NULL));
1741 
1742 	napi_enable(&cq->napi);
1743 
1744 	mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT);
1745 out:
1746 	if (!err)
1747 		return rxq;
1748 
1749 	netdev_err(ndev, "Failed to create RXQ: err = %d\n", err);
1750 
1751 	mana_destroy_rxq(apc, rxq, false);
1752 
1753 	if (cq)
1754 		mana_deinit_cq(apc, cq);
1755 
1756 	return NULL;
1757 }
1758 
mana_add_rx_queues(struct mana_port_context * apc,struct net_device * ndev)1759 static int mana_add_rx_queues(struct mana_port_context *apc,
1760 			      struct net_device *ndev)
1761 {
1762 	struct mana_context *ac = apc->ac;
1763 	struct mana_rxq *rxq;
1764 	int err = 0;
1765 	int i;
1766 
1767 	for (i = 0; i < apc->num_queues; i++) {
1768 		rxq = mana_create_rxq(apc, i, &ac->eqs[i], ndev);
1769 		if (!rxq) {
1770 			err = -ENOMEM;
1771 			goto out;
1772 		}
1773 
1774 		u64_stats_init(&rxq->stats.syncp);
1775 
1776 		apc->rxqs[i] = rxq;
1777 	}
1778 
1779 	apc->default_rxobj = apc->rxqs[0]->rxobj;
1780 out:
1781 	return err;
1782 }
1783 
mana_destroy_vport(struct mana_port_context * apc)1784 static void mana_destroy_vport(struct mana_port_context *apc)
1785 {
1786 	struct gdma_dev *gd = apc->ac->gdma_dev;
1787 	struct mana_rxq *rxq;
1788 	u32 rxq_idx;
1789 
1790 	for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) {
1791 		rxq = apc->rxqs[rxq_idx];
1792 		if (!rxq)
1793 			continue;
1794 
1795 		mana_destroy_rxq(apc, rxq, true);
1796 		apc->rxqs[rxq_idx] = NULL;
1797 	}
1798 
1799 	mana_destroy_txq(apc);
1800 
1801 	if (gd->gdma_context->is_pf)
1802 		mana_pf_deregister_hw_vport(apc);
1803 }
1804 
mana_create_vport(struct mana_port_context * apc,struct net_device * net)1805 static int mana_create_vport(struct mana_port_context *apc,
1806 			     struct net_device *net)
1807 {
1808 	struct gdma_dev *gd = apc->ac->gdma_dev;
1809 	int err;
1810 
1811 	apc->default_rxobj = INVALID_MANA_HANDLE;
1812 
1813 	if (gd->gdma_context->is_pf) {
1814 		err = mana_pf_register_hw_vport(apc);
1815 		if (err)
1816 			return err;
1817 	}
1818 
1819 	err = mana_cfg_vport(apc, gd->pdid, gd->doorbell);
1820 	if (err)
1821 		return err;
1822 
1823 	return mana_create_txq(apc, net);
1824 }
1825 
mana_rss_table_init(struct mana_port_context * apc)1826 static void mana_rss_table_init(struct mana_port_context *apc)
1827 {
1828 	int i;
1829 
1830 	for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++)
1831 		apc->indir_table[i] =
1832 			ethtool_rxfh_indir_default(i, apc->num_queues);
1833 }
1834 
mana_config_rss(struct mana_port_context * apc,enum TRI_STATE rx,bool update_hash,bool update_tab)1835 int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx,
1836 		    bool update_hash, bool update_tab)
1837 {
1838 	u32 queue_idx;
1839 	int err;
1840 	int i;
1841 
1842 	if (update_tab) {
1843 		for (i = 0; i < MANA_INDIRECT_TABLE_SIZE; i++) {
1844 			queue_idx = apc->indir_table[i];
1845 			apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj;
1846 		}
1847 	}
1848 
1849 	err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab);
1850 	if (err)
1851 		return err;
1852 
1853 	mana_fence_rqs(apc);
1854 
1855 	return 0;
1856 }
1857 
mana_init_port(struct net_device * ndev)1858 static int mana_init_port(struct net_device *ndev)
1859 {
1860 	struct mana_port_context *apc = netdev_priv(ndev);
1861 	u32 max_txq, max_rxq, max_queues;
1862 	int port_idx = apc->port_idx;
1863 	u32 num_indirect_entries;
1864 	int err;
1865 
1866 	err = mana_init_port_context(apc);
1867 	if (err)
1868 		return err;
1869 
1870 	err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq,
1871 				   &num_indirect_entries);
1872 	if (err) {
1873 		netdev_err(ndev, "Failed to query info for vPort %d\n",
1874 			   port_idx);
1875 		goto reset_apc;
1876 	}
1877 
1878 	max_queues = min_t(u32, max_txq, max_rxq);
1879 	if (apc->max_queues > max_queues)
1880 		apc->max_queues = max_queues;
1881 
1882 	if (apc->num_queues > apc->max_queues)
1883 		apc->num_queues = apc->max_queues;
1884 
1885 	eth_hw_addr_set(ndev, apc->mac_addr);
1886 
1887 	return 0;
1888 
1889 reset_apc:
1890 	kfree(apc->rxqs);
1891 	apc->rxqs = NULL;
1892 	return err;
1893 }
1894 
mana_alloc_queues(struct net_device * ndev)1895 int mana_alloc_queues(struct net_device *ndev)
1896 {
1897 	struct mana_port_context *apc = netdev_priv(ndev);
1898 	struct gdma_dev *gd = apc->ac->gdma_dev;
1899 	int err;
1900 
1901 	err = mana_create_vport(apc, ndev);
1902 	if (err)
1903 		return err;
1904 
1905 	err = netif_set_real_num_tx_queues(ndev, apc->num_queues);
1906 	if (err)
1907 		goto destroy_vport;
1908 
1909 	err = mana_add_rx_queues(apc, ndev);
1910 	if (err)
1911 		goto destroy_vport;
1912 
1913 	apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE;
1914 
1915 	err = netif_set_real_num_rx_queues(ndev, apc->num_queues);
1916 	if (err)
1917 		goto destroy_vport;
1918 
1919 	mana_rss_table_init(apc);
1920 
1921 	err = mana_config_rss(apc, TRI_STATE_TRUE, true, true);
1922 	if (err)
1923 		goto destroy_vport;
1924 
1925 	if (gd->gdma_context->is_pf) {
1926 		err = mana_pf_register_filter(apc);
1927 		if (err)
1928 			goto destroy_vport;
1929 	}
1930 
1931 	mana_chn_setxdp(apc, mana_xdp_get(apc));
1932 
1933 	return 0;
1934 
1935 destroy_vport:
1936 	mana_destroy_vport(apc);
1937 	return err;
1938 }
1939 
mana_attach(struct net_device * ndev)1940 int mana_attach(struct net_device *ndev)
1941 {
1942 	struct mana_port_context *apc = netdev_priv(ndev);
1943 	int err;
1944 
1945 	ASSERT_RTNL();
1946 
1947 	err = mana_init_port(ndev);
1948 	if (err)
1949 		return err;
1950 
1951 	if (apc->port_st_save) {
1952 		err = mana_alloc_queues(ndev);
1953 		if (err) {
1954 			mana_cleanup_port_context(apc);
1955 			return err;
1956 		}
1957 	}
1958 
1959 	apc->port_is_up = apc->port_st_save;
1960 
1961 	/* Ensure port state updated before txq state */
1962 	smp_wmb();
1963 
1964 	if (apc->port_is_up)
1965 		netif_carrier_on(ndev);
1966 
1967 	netif_device_attach(ndev);
1968 
1969 	return 0;
1970 }
1971 
mana_dealloc_queues(struct net_device * ndev)1972 static int mana_dealloc_queues(struct net_device *ndev)
1973 {
1974 	struct mana_port_context *apc = netdev_priv(ndev);
1975 	struct gdma_dev *gd = apc->ac->gdma_dev;
1976 	struct mana_txq *txq;
1977 	int i, err;
1978 
1979 	if (apc->port_is_up)
1980 		return -EINVAL;
1981 
1982 	mana_chn_setxdp(apc, NULL);
1983 
1984 	if (gd->gdma_context->is_pf)
1985 		mana_pf_deregister_filter(apc);
1986 
1987 	/* No packet can be transmitted now since apc->port_is_up is false.
1988 	 * There is still a tiny chance that mana_poll_tx_cq() can re-enable
1989 	 * a txq because it may not timely see apc->port_is_up being cleared
1990 	 * to false, but it doesn't matter since mana_start_xmit() drops any
1991 	 * new packets due to apc->port_is_up being false.
1992 	 *
1993 	 * Drain all the in-flight TX packets
1994 	 */
1995 	for (i = 0; i < apc->num_queues; i++) {
1996 		txq = &apc->tx_qp[i].txq;
1997 
1998 		while (atomic_read(&txq->pending_sends) > 0)
1999 			usleep_range(1000, 2000);
2000 	}
2001 
2002 	/* We're 100% sure the queues can no longer be woken up, because
2003 	 * we're sure now mana_poll_tx_cq() can't be running.
2004 	 */
2005 
2006 	apc->rss_state = TRI_STATE_FALSE;
2007 	err = mana_config_rss(apc, TRI_STATE_FALSE, false, false);
2008 	if (err) {
2009 		netdev_err(ndev, "Failed to disable vPort: %d\n", err);
2010 		return err;
2011 	}
2012 
2013 	mana_destroy_vport(apc);
2014 
2015 	return 0;
2016 }
2017 
mana_detach(struct net_device * ndev,bool from_close)2018 int mana_detach(struct net_device *ndev, bool from_close)
2019 {
2020 	struct mana_port_context *apc = netdev_priv(ndev);
2021 	int err;
2022 
2023 	ASSERT_RTNL();
2024 
2025 	apc->port_st_save = apc->port_is_up;
2026 	apc->port_is_up = false;
2027 
2028 	/* Ensure port state updated before txq state */
2029 	smp_wmb();
2030 
2031 	netif_tx_disable(ndev);
2032 	netif_carrier_off(ndev);
2033 
2034 	if (apc->port_st_save) {
2035 		err = mana_dealloc_queues(ndev);
2036 		if (err)
2037 			return err;
2038 	}
2039 
2040 	if (!from_close) {
2041 		netif_device_detach(ndev);
2042 		mana_cleanup_port_context(apc);
2043 	}
2044 
2045 	return 0;
2046 }
2047 
mana_probe_port(struct mana_context * ac,int port_idx,struct net_device ** ndev_storage)2048 static int mana_probe_port(struct mana_context *ac, int port_idx,
2049 			   struct net_device **ndev_storage)
2050 {
2051 	struct gdma_context *gc = ac->gdma_dev->gdma_context;
2052 	struct mana_port_context *apc;
2053 	struct net_device *ndev;
2054 	int err;
2055 
2056 	ndev = alloc_etherdev_mq(sizeof(struct mana_port_context),
2057 				 gc->max_num_queues);
2058 	if (!ndev)
2059 		return -ENOMEM;
2060 
2061 	*ndev_storage = ndev;
2062 
2063 	apc = netdev_priv(ndev);
2064 	apc->ac = ac;
2065 	apc->ndev = ndev;
2066 	apc->max_queues = gc->max_num_queues;
2067 	apc->num_queues = gc->max_num_queues;
2068 	apc->port_handle = INVALID_MANA_HANDLE;
2069 	apc->pf_filter_handle = INVALID_MANA_HANDLE;
2070 	apc->port_idx = port_idx;
2071 
2072 	ndev->netdev_ops = &mana_devops;
2073 	ndev->ethtool_ops = &mana_ethtool_ops;
2074 	ndev->mtu = ETH_DATA_LEN;
2075 	ndev->max_mtu = ndev->mtu;
2076 	ndev->min_mtu = ndev->mtu;
2077 	ndev->needed_headroom = MANA_HEADROOM;
2078 	SET_NETDEV_DEV(ndev, gc->dev);
2079 
2080 	netif_carrier_off(ndev);
2081 
2082 	netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE);
2083 
2084 	err = mana_init_port(ndev);
2085 	if (err)
2086 		goto free_net;
2087 
2088 	netdev_lockdep_set_classes(ndev);
2089 
2090 	ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM;
2091 	ndev->hw_features |= NETIF_F_RXCSUM;
2092 	ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
2093 	ndev->hw_features |= NETIF_F_RXHASH;
2094 	ndev->features = ndev->hw_features;
2095 	ndev->vlan_features = 0;
2096 
2097 	err = register_netdev(ndev);
2098 	if (err) {
2099 		netdev_err(ndev, "Unable to register netdev.\n");
2100 		goto reset_apc;
2101 	}
2102 
2103 	return 0;
2104 
2105 reset_apc:
2106 	kfree(apc->rxqs);
2107 	apc->rxqs = NULL;
2108 free_net:
2109 	*ndev_storage = NULL;
2110 	netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err);
2111 	free_netdev(ndev);
2112 	return err;
2113 }
2114 
mana_probe(struct gdma_dev * gd,bool resuming)2115 int mana_probe(struct gdma_dev *gd, bool resuming)
2116 {
2117 	struct gdma_context *gc = gd->gdma_context;
2118 	struct mana_context *ac = gd->driver_data;
2119 	struct device *dev = gc->dev;
2120 	u16 num_ports = 0;
2121 	int err;
2122 	int i;
2123 
2124 	dev_info(dev,
2125 		 "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n",
2126 		 MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION);
2127 
2128 	err = mana_gd_register_device(gd);
2129 	if (err)
2130 		return err;
2131 
2132 	if (!resuming) {
2133 		ac = kzalloc(sizeof(*ac), GFP_KERNEL);
2134 		if (!ac)
2135 			return -ENOMEM;
2136 
2137 		ac->gdma_dev = gd;
2138 		gd->driver_data = ac;
2139 	}
2140 
2141 	err = mana_create_eq(ac);
2142 	if (err)
2143 		goto out;
2144 
2145 	err = mana_query_device_cfg(ac, MANA_MAJOR_VERSION, MANA_MINOR_VERSION,
2146 				    MANA_MICRO_VERSION, &num_ports);
2147 	if (err)
2148 		goto out;
2149 
2150 	if (!resuming) {
2151 		ac->num_ports = num_ports;
2152 	} else {
2153 		if (ac->num_ports != num_ports) {
2154 			dev_err(dev, "The number of vPorts changed: %d->%d\n",
2155 				ac->num_ports, num_ports);
2156 			err = -EPROTO;
2157 			goto out;
2158 		}
2159 	}
2160 
2161 	if (ac->num_ports == 0)
2162 		dev_err(dev, "Failed to detect any vPort\n");
2163 
2164 	if (ac->num_ports > MAX_PORTS_IN_MANA_DEV)
2165 		ac->num_ports = MAX_PORTS_IN_MANA_DEV;
2166 
2167 	if (!resuming) {
2168 		for (i = 0; i < ac->num_ports; i++) {
2169 			err = mana_probe_port(ac, i, &ac->ports[i]);
2170 			if (err)
2171 				break;
2172 		}
2173 	} else {
2174 		for (i = 0; i < ac->num_ports; i++) {
2175 			rtnl_lock();
2176 			err = mana_attach(ac->ports[i]);
2177 			rtnl_unlock();
2178 			if (err)
2179 				break;
2180 		}
2181 	}
2182 out:
2183 	if (err)
2184 		mana_remove(gd, false);
2185 
2186 	return err;
2187 }
2188 
mana_remove(struct gdma_dev * gd,bool suspending)2189 void mana_remove(struct gdma_dev *gd, bool suspending)
2190 {
2191 	struct gdma_context *gc = gd->gdma_context;
2192 	struct mana_context *ac = gd->driver_data;
2193 	struct device *dev = gc->dev;
2194 	struct net_device *ndev;
2195 	int err;
2196 	int i;
2197 
2198 	for (i = 0; i < ac->num_ports; i++) {
2199 		ndev = ac->ports[i];
2200 		if (!ndev) {
2201 			if (i == 0)
2202 				dev_err(dev, "No net device to remove\n");
2203 			goto out;
2204 		}
2205 
2206 		/* All cleanup actions should stay after rtnl_lock(), otherwise
2207 		 * other functions may access partially cleaned up data.
2208 		 */
2209 		rtnl_lock();
2210 
2211 		err = mana_detach(ndev, false);
2212 		if (err)
2213 			netdev_err(ndev, "Failed to detach vPort %d: %d\n",
2214 				   i, err);
2215 
2216 		if (suspending) {
2217 			/* No need to unregister the ndev. */
2218 			rtnl_unlock();
2219 			continue;
2220 		}
2221 
2222 		unregister_netdevice(ndev);
2223 
2224 		rtnl_unlock();
2225 
2226 		free_netdev(ndev);
2227 	}
2228 
2229 	mana_destroy_eq(ac);
2230 
2231 out:
2232 	mana_gd_deregister_device(gd);
2233 
2234 	if (suspending)
2235 		return;
2236 
2237 	gd->driver_data = NULL;
2238 	gd->gdma_context = NULL;
2239 	kfree(ac);
2240 }
2241