1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "main.h"
4
5 /**
6 * irdma_arp_table -manage arp table
7 * @rf: RDMA PCI function
8 * @ip_addr: ip address for device
9 * @ipv4: IPv4 flag
10 * @mac_addr: mac address ptr
11 * @action: modify, delete or add
12 */
irdma_arp_table(struct irdma_pci_f * rf,u32 * ip_addr,bool ipv4,const u8 * mac_addr,u32 action)13 int irdma_arp_table(struct irdma_pci_f *rf, u32 *ip_addr, bool ipv4,
14 const u8 *mac_addr, u32 action)
15 {
16 unsigned long flags;
17 int arp_index;
18 u32 ip[4] = {};
19
20 if (ipv4)
21 ip[0] = *ip_addr;
22 else
23 memcpy(ip, ip_addr, sizeof(ip));
24
25 spin_lock_irqsave(&rf->arp_lock, flags);
26 for (arp_index = 0; (u32)arp_index < rf->arp_table_size; arp_index++) {
27 if (!memcmp(rf->arp_table[arp_index].ip_addr, ip, sizeof(ip)))
28 break;
29 }
30
31 switch (action) {
32 case IRDMA_ARP_ADD:
33 if (arp_index != rf->arp_table_size) {
34 arp_index = -1;
35 break;
36 }
37
38 arp_index = 0;
39 if (irdma_alloc_rsrc(rf, rf->allocated_arps, rf->arp_table_size,
40 (u32 *)&arp_index, &rf->next_arp_index)) {
41 arp_index = -1;
42 break;
43 }
44
45 memcpy(rf->arp_table[arp_index].ip_addr, ip,
46 sizeof(rf->arp_table[arp_index].ip_addr));
47 ether_addr_copy(rf->arp_table[arp_index].mac_addr, mac_addr);
48 break;
49 case IRDMA_ARP_RESOLVE:
50 if (arp_index == rf->arp_table_size)
51 arp_index = -1;
52 break;
53 case IRDMA_ARP_DELETE:
54 if (arp_index == rf->arp_table_size) {
55 arp_index = -1;
56 break;
57 }
58
59 memset(rf->arp_table[arp_index].ip_addr, 0,
60 sizeof(rf->arp_table[arp_index].ip_addr));
61 eth_zero_addr(rf->arp_table[arp_index].mac_addr);
62 irdma_free_rsrc(rf, rf->allocated_arps, arp_index);
63 break;
64 default:
65 arp_index = -1;
66 break;
67 }
68
69 spin_unlock_irqrestore(&rf->arp_lock, flags);
70 return arp_index;
71 }
72
73 /**
74 * irdma_add_arp - add a new arp entry if needed
75 * @rf: RDMA function
76 * @ip: IP address
77 * @ipv4: IPv4 flag
78 * @mac: MAC address
79 */
irdma_add_arp(struct irdma_pci_f * rf,u32 * ip,bool ipv4,const u8 * mac)80 int irdma_add_arp(struct irdma_pci_f *rf, u32 *ip, bool ipv4, const u8 *mac)
81 {
82 int arpidx;
83
84 arpidx = irdma_arp_table(rf, &ip[0], ipv4, NULL, IRDMA_ARP_RESOLVE);
85 if (arpidx >= 0) {
86 if (ether_addr_equal(rf->arp_table[arpidx].mac_addr, mac))
87 return arpidx;
88
89 irdma_manage_arp_cache(rf, rf->arp_table[arpidx].mac_addr, ip,
90 ipv4, IRDMA_ARP_DELETE);
91 }
92
93 irdma_manage_arp_cache(rf, mac, ip, ipv4, IRDMA_ARP_ADD);
94
95 return irdma_arp_table(rf, ip, ipv4, NULL, IRDMA_ARP_RESOLVE);
96 }
97
98 /**
99 * wr32 - write 32 bits to hw register
100 * @hw: hardware information including registers
101 * @reg: register offset
102 * @val: value to write to register
103 */
wr32(struct irdma_hw * hw,u32 reg,u32 val)104 inline void wr32(struct irdma_hw *hw, u32 reg, u32 val)
105 {
106 writel(val, hw->hw_addr + reg);
107 }
108
109 /**
110 * rd32 - read a 32 bit hw register
111 * @hw: hardware information including registers
112 * @reg: register offset
113 *
114 * Return value of register content
115 */
rd32(struct irdma_hw * hw,u32 reg)116 inline u32 rd32(struct irdma_hw *hw, u32 reg)
117 {
118 return readl(hw->hw_addr + reg);
119 }
120
121 /**
122 * rd64 - read a 64 bit hw register
123 * @hw: hardware information including registers
124 * @reg: register offset
125 *
126 * Return value of register content
127 */
rd64(struct irdma_hw * hw,u32 reg)128 inline u64 rd64(struct irdma_hw *hw, u32 reg)
129 {
130 return readq(hw->hw_addr + reg);
131 }
132
irdma_gid_change_event(struct ib_device * ibdev)133 static void irdma_gid_change_event(struct ib_device *ibdev)
134 {
135 struct ib_event ib_event;
136
137 ib_event.event = IB_EVENT_GID_CHANGE;
138 ib_event.device = ibdev;
139 ib_event.element.port_num = 1;
140 ib_dispatch_event(&ib_event);
141 }
142
143 /**
144 * irdma_inetaddr_event - system notifier for ipv4 addr events
145 * @notifier: not used
146 * @event: event for notifier
147 * @ptr: if address
148 */
irdma_inetaddr_event(struct notifier_block * notifier,unsigned long event,void * ptr)149 int irdma_inetaddr_event(struct notifier_block *notifier, unsigned long event,
150 void *ptr)
151 {
152 struct in_ifaddr *ifa = ptr;
153 struct net_device *real_dev, *netdev = ifa->ifa_dev->dev;
154 struct irdma_device *iwdev;
155 struct ib_device *ibdev;
156 u32 local_ipaddr;
157
158 real_dev = rdma_vlan_dev_real_dev(netdev);
159 if (!real_dev)
160 real_dev = netdev;
161
162 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
163 if (!ibdev)
164 return NOTIFY_DONE;
165
166 iwdev = to_iwdev(ibdev);
167 local_ipaddr = ntohl(ifa->ifa_address);
168 ibdev_dbg(&iwdev->ibdev,
169 "DEV: netdev %p event %lu local_ip=%pI4 MAC=%pM\n", real_dev,
170 event, &local_ipaddr, real_dev->dev_addr);
171 switch (event) {
172 case NETDEV_DOWN:
173 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
174 &local_ipaddr, true, IRDMA_ARP_DELETE);
175 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, false);
176 irdma_gid_change_event(&iwdev->ibdev);
177 break;
178 case NETDEV_UP:
179 case NETDEV_CHANGEADDR:
180 irdma_add_arp(iwdev->rf, &local_ipaddr, true, real_dev->dev_addr);
181 irdma_if_notify(iwdev, real_dev, &local_ipaddr, true, true);
182 irdma_gid_change_event(&iwdev->ibdev);
183 break;
184 default:
185 break;
186 }
187
188 ib_device_put(ibdev);
189
190 return NOTIFY_DONE;
191 }
192
193 /**
194 * irdma_inet6addr_event - system notifier for ipv6 addr events
195 * @notifier: not used
196 * @event: event for notifier
197 * @ptr: if address
198 */
irdma_inet6addr_event(struct notifier_block * notifier,unsigned long event,void * ptr)199 int irdma_inet6addr_event(struct notifier_block *notifier, unsigned long event,
200 void *ptr)
201 {
202 struct inet6_ifaddr *ifa = ptr;
203 struct net_device *real_dev, *netdev = ifa->idev->dev;
204 struct irdma_device *iwdev;
205 struct ib_device *ibdev;
206 u32 local_ipaddr6[4];
207
208 real_dev = rdma_vlan_dev_real_dev(netdev);
209 if (!real_dev)
210 real_dev = netdev;
211
212 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
213 if (!ibdev)
214 return NOTIFY_DONE;
215
216 iwdev = to_iwdev(ibdev);
217 irdma_copy_ip_ntohl(local_ipaddr6, ifa->addr.in6_u.u6_addr32);
218 ibdev_dbg(&iwdev->ibdev,
219 "DEV: netdev %p event %lu local_ip=%pI6 MAC=%pM\n", real_dev,
220 event, local_ipaddr6, real_dev->dev_addr);
221 switch (event) {
222 case NETDEV_DOWN:
223 irdma_manage_arp_cache(iwdev->rf, real_dev->dev_addr,
224 local_ipaddr6, false, IRDMA_ARP_DELETE);
225 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, false);
226 irdma_gid_change_event(&iwdev->ibdev);
227 break;
228 case NETDEV_UP:
229 case NETDEV_CHANGEADDR:
230 irdma_add_arp(iwdev->rf, local_ipaddr6, false,
231 real_dev->dev_addr);
232 irdma_if_notify(iwdev, real_dev, local_ipaddr6, false, true);
233 irdma_gid_change_event(&iwdev->ibdev);
234 break;
235 default:
236 break;
237 }
238
239 ib_device_put(ibdev);
240
241 return NOTIFY_DONE;
242 }
243
244 /**
245 * irdma_net_event - system notifier for net events
246 * @notifier: not used
247 * @event: event for notifier
248 * @ptr: neighbor
249 */
irdma_net_event(struct notifier_block * notifier,unsigned long event,void * ptr)250 int irdma_net_event(struct notifier_block *notifier, unsigned long event,
251 void *ptr)
252 {
253 struct neighbour *neigh = ptr;
254 struct net_device *real_dev, *netdev = (struct net_device *)neigh->dev;
255 struct irdma_device *iwdev;
256 struct ib_device *ibdev;
257 __be32 *p;
258 u32 local_ipaddr[4] = {};
259 bool ipv4 = true;
260
261 switch (event) {
262 case NETEVENT_NEIGH_UPDATE:
263 real_dev = rdma_vlan_dev_real_dev(netdev);
264 if (!real_dev)
265 real_dev = netdev;
266 ibdev = ib_device_get_by_netdev(real_dev, RDMA_DRIVER_IRDMA);
267 if (!ibdev)
268 return NOTIFY_DONE;
269
270 iwdev = to_iwdev(ibdev);
271 p = (__be32 *)neigh->primary_key;
272 if (neigh->tbl->family == AF_INET6) {
273 ipv4 = false;
274 irdma_copy_ip_ntohl(local_ipaddr, p);
275 } else {
276 local_ipaddr[0] = ntohl(*p);
277 }
278
279 ibdev_dbg(&iwdev->ibdev,
280 "DEV: netdev %p state %d local_ip=%pI4 MAC=%pM\n",
281 iwdev->netdev, neigh->nud_state, local_ipaddr,
282 neigh->ha);
283
284 if (neigh->nud_state & NUD_VALID)
285 irdma_add_arp(iwdev->rf, local_ipaddr, ipv4, neigh->ha);
286
287 else
288 irdma_manage_arp_cache(iwdev->rf, neigh->ha,
289 local_ipaddr, ipv4,
290 IRDMA_ARP_DELETE);
291 ib_device_put(ibdev);
292 break;
293 default:
294 break;
295 }
296
297 return NOTIFY_DONE;
298 }
299
300 /**
301 * irdma_netdevice_event - system notifier for netdev events
302 * @notifier: not used
303 * @event: event for notifier
304 * @ptr: netdev
305 */
irdma_netdevice_event(struct notifier_block * notifier,unsigned long event,void * ptr)306 int irdma_netdevice_event(struct notifier_block *notifier, unsigned long event,
307 void *ptr)
308 {
309 struct irdma_device *iwdev;
310 struct ib_device *ibdev;
311 struct net_device *netdev = netdev_notifier_info_to_dev(ptr);
312
313 ibdev = ib_device_get_by_netdev(netdev, RDMA_DRIVER_IRDMA);
314 if (!ibdev)
315 return NOTIFY_DONE;
316
317 iwdev = to_iwdev(ibdev);
318 iwdev->iw_status = 1;
319 switch (event) {
320 case NETDEV_DOWN:
321 iwdev->iw_status = 0;
322 fallthrough;
323 case NETDEV_UP:
324 irdma_port_ibevent(iwdev);
325 break;
326 default:
327 break;
328 }
329 ib_device_put(ibdev);
330
331 return NOTIFY_DONE;
332 }
333
334 /**
335 * irdma_add_ipv6_addr - add ipv6 address to the hw arp table
336 * @iwdev: irdma device
337 */
irdma_add_ipv6_addr(struct irdma_device * iwdev)338 static void irdma_add_ipv6_addr(struct irdma_device *iwdev)
339 {
340 struct net_device *ip_dev;
341 struct inet6_dev *idev;
342 struct inet6_ifaddr *ifp, *tmp;
343 u32 local_ipaddr6[4];
344
345 rcu_read_lock();
346 for_each_netdev_rcu (&init_net, ip_dev) {
347 if (((rdma_vlan_dev_vlan_id(ip_dev) < 0xFFFF &&
348 rdma_vlan_dev_real_dev(ip_dev) == iwdev->netdev) ||
349 ip_dev == iwdev->netdev) &&
350 (READ_ONCE(ip_dev->flags) & IFF_UP)) {
351 idev = __in6_dev_get(ip_dev);
352 if (!idev) {
353 ibdev_err(&iwdev->ibdev, "ipv6 inet device not found\n");
354 break;
355 }
356 list_for_each_entry_safe (ifp, tmp, &idev->addr_list,
357 if_list) {
358 ibdev_dbg(&iwdev->ibdev,
359 "INIT: IP=%pI6, vlan_id=%d, MAC=%pM\n",
360 &ifp->addr,
361 rdma_vlan_dev_vlan_id(ip_dev),
362 ip_dev->dev_addr);
363
364 irdma_copy_ip_ntohl(local_ipaddr6,
365 ifp->addr.in6_u.u6_addr32);
366 irdma_manage_arp_cache(iwdev->rf,
367 ip_dev->dev_addr,
368 local_ipaddr6, false,
369 IRDMA_ARP_ADD);
370 }
371 }
372 }
373 rcu_read_unlock();
374 }
375
376 /**
377 * irdma_add_ipv4_addr - add ipv4 address to the hw arp table
378 * @iwdev: irdma device
379 */
irdma_add_ipv4_addr(struct irdma_device * iwdev)380 static void irdma_add_ipv4_addr(struct irdma_device *iwdev)
381 {
382 struct net_device *dev;
383 struct in_device *idev;
384 u32 ip_addr;
385
386 rcu_read_lock();
387 for_each_netdev_rcu (&init_net, dev) {
388 if (((rdma_vlan_dev_vlan_id(dev) < 0xFFFF &&
389 rdma_vlan_dev_real_dev(dev) == iwdev->netdev) ||
390 dev == iwdev->netdev) && (READ_ONCE(dev->flags) & IFF_UP)) {
391 const struct in_ifaddr *ifa;
392
393 idev = __in_dev_get_rcu(dev);
394 if (!idev)
395 continue;
396
397 in_dev_for_each_ifa_rcu(ifa, idev) {
398 ibdev_dbg(&iwdev->ibdev, "CM: IP=%pI4, vlan_id=%d, MAC=%pM\n",
399 &ifa->ifa_address, rdma_vlan_dev_vlan_id(dev),
400 dev->dev_addr);
401
402 ip_addr = ntohl(ifa->ifa_address);
403 irdma_manage_arp_cache(iwdev->rf, dev->dev_addr,
404 &ip_addr, true,
405 IRDMA_ARP_ADD);
406 }
407 }
408 }
409 rcu_read_unlock();
410 }
411
412 /**
413 * irdma_add_ip - add ip addresses
414 * @iwdev: irdma device
415 *
416 * Add ipv4/ipv6 addresses to the arp cache
417 */
irdma_add_ip(struct irdma_device * iwdev)418 void irdma_add_ip(struct irdma_device *iwdev)
419 {
420 irdma_add_ipv4_addr(iwdev);
421 irdma_add_ipv6_addr(iwdev);
422 }
423
424 /**
425 * irdma_alloc_and_get_cqp_request - get cqp struct
426 * @cqp: device cqp ptr
427 * @wait: cqp to be used in wait mode
428 */
irdma_alloc_and_get_cqp_request(struct irdma_cqp * cqp,bool wait)429 struct irdma_cqp_request *irdma_alloc_and_get_cqp_request(struct irdma_cqp *cqp,
430 bool wait)
431 {
432 struct irdma_cqp_request *cqp_request = NULL;
433 unsigned long flags;
434
435 spin_lock_irqsave(&cqp->req_lock, flags);
436 if (!list_empty(&cqp->cqp_avail_reqs)) {
437 cqp_request = list_first_entry(&cqp->cqp_avail_reqs,
438 struct irdma_cqp_request, list);
439 list_del_init(&cqp_request->list);
440 }
441 spin_unlock_irqrestore(&cqp->req_lock, flags);
442 if (!cqp_request) {
443 cqp_request = kzalloc(sizeof(*cqp_request), GFP_ATOMIC);
444 if (cqp_request) {
445 cqp_request->dynamic = true;
446 if (wait)
447 init_waitqueue_head(&cqp_request->waitq);
448 }
449 }
450 if (!cqp_request) {
451 ibdev_dbg(to_ibdev(cqp->sc_cqp.dev), "ERR: CQP Request Fail: No Memory");
452 return NULL;
453 }
454
455 cqp_request->waiting = wait;
456 refcount_set(&cqp_request->refcnt, 1);
457 memset(&cqp_request->compl_info, 0, sizeof(cqp_request->compl_info));
458
459 return cqp_request;
460 }
461
462 /**
463 * irdma_get_cqp_request - increase refcount for cqp_request
464 * @cqp_request: pointer to cqp_request instance
465 */
irdma_get_cqp_request(struct irdma_cqp_request * cqp_request)466 static inline void irdma_get_cqp_request(struct irdma_cqp_request *cqp_request)
467 {
468 refcount_inc(&cqp_request->refcnt);
469 }
470
471 /**
472 * irdma_free_cqp_request - free cqp request
473 * @cqp: cqp ptr
474 * @cqp_request: to be put back in cqp list
475 */
irdma_free_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)476 void irdma_free_cqp_request(struct irdma_cqp *cqp,
477 struct irdma_cqp_request *cqp_request)
478 {
479 unsigned long flags;
480
481 if (cqp_request->dynamic) {
482 kfree(cqp_request);
483 } else {
484 cqp_request->request_done = false;
485 cqp_request->callback_fcn = NULL;
486 cqp_request->waiting = false;
487
488 spin_lock_irqsave(&cqp->req_lock, flags);
489 list_add_tail(&cqp_request->list, &cqp->cqp_avail_reqs);
490 spin_unlock_irqrestore(&cqp->req_lock, flags);
491 }
492 wake_up(&cqp->remove_wq);
493 }
494
495 /**
496 * irdma_put_cqp_request - dec ref count and free if 0
497 * @cqp: cqp ptr
498 * @cqp_request: to be put back in cqp list
499 */
irdma_put_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)500 void irdma_put_cqp_request(struct irdma_cqp *cqp,
501 struct irdma_cqp_request *cqp_request)
502 {
503 if (refcount_dec_and_test(&cqp_request->refcnt))
504 irdma_free_cqp_request(cqp, cqp_request);
505 }
506
507 /**
508 * irdma_free_pending_cqp_request -free pending cqp request objs
509 * @cqp: cqp ptr
510 * @cqp_request: to be put back in cqp list
511 */
512 static void
irdma_free_pending_cqp_request(struct irdma_cqp * cqp,struct irdma_cqp_request * cqp_request)513 irdma_free_pending_cqp_request(struct irdma_cqp *cqp,
514 struct irdma_cqp_request *cqp_request)
515 {
516 if (cqp_request->waiting) {
517 cqp_request->compl_info.error = true;
518 cqp_request->request_done = true;
519 wake_up(&cqp_request->waitq);
520 }
521 wait_event_timeout(cqp->remove_wq,
522 refcount_read(&cqp_request->refcnt) == 1, 1000);
523 irdma_put_cqp_request(cqp, cqp_request);
524 }
525
526 /**
527 * irdma_cleanup_pending_cqp_op - clean-up cqp with no
528 * completions
529 * @rf: RDMA PCI function
530 */
irdma_cleanup_pending_cqp_op(struct irdma_pci_f * rf)531 void irdma_cleanup_pending_cqp_op(struct irdma_pci_f *rf)
532 {
533 struct irdma_sc_dev *dev = &rf->sc_dev;
534 struct irdma_cqp *cqp = &rf->cqp;
535 struct irdma_cqp_request *cqp_request = NULL;
536 struct cqp_cmds_info *pcmdinfo = NULL;
537 u32 i, pending_work, wqe_idx;
538
539 pending_work = IRDMA_RING_USED_QUANTA(cqp->sc_cqp.sq_ring);
540 wqe_idx = IRDMA_RING_CURRENT_TAIL(cqp->sc_cqp.sq_ring);
541 for (i = 0; i < pending_work; i++) {
542 cqp_request = (struct irdma_cqp_request *)(unsigned long)
543 cqp->scratch_array[wqe_idx];
544 if (cqp_request)
545 irdma_free_pending_cqp_request(cqp, cqp_request);
546 wqe_idx = (wqe_idx + 1) % IRDMA_RING_SIZE(cqp->sc_cqp.sq_ring);
547 }
548
549 while (!list_empty(&dev->cqp_cmd_head)) {
550 pcmdinfo = irdma_remove_cqp_head(dev);
551 cqp_request =
552 container_of(pcmdinfo, struct irdma_cqp_request, info);
553 if (cqp_request)
554 irdma_free_pending_cqp_request(cqp, cqp_request);
555 }
556 }
557
558 /**
559 * irdma_wait_event - wait for completion
560 * @rf: RDMA PCI function
561 * @cqp_request: cqp request to wait
562 */
irdma_wait_event(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)563 static int irdma_wait_event(struct irdma_pci_f *rf,
564 struct irdma_cqp_request *cqp_request)
565 {
566 struct irdma_cqp_timeout cqp_timeout = {};
567 bool cqp_error = false;
568 int err_code = 0;
569
570 cqp_timeout.compl_cqp_cmds = rf->sc_dev.cqp_cmd_stats[IRDMA_OP_CMPL_CMDS];
571 do {
572 irdma_cqp_ce_handler(rf, &rf->ccq.sc_cq);
573 if (wait_event_timeout(cqp_request->waitq,
574 cqp_request->request_done,
575 msecs_to_jiffies(CQP_COMPL_WAIT_TIME_MS)))
576 break;
577
578 irdma_check_cqp_progress(&cqp_timeout, &rf->sc_dev);
579
580 if (cqp_timeout.count < CQP_TIMEOUT_THRESHOLD)
581 continue;
582
583 if (!rf->reset) {
584 rf->reset = true;
585 rf->gen_ops.request_reset(rf);
586 }
587 return -ETIMEDOUT;
588 } while (1);
589
590 cqp_error = cqp_request->compl_info.error;
591 if (cqp_error) {
592 err_code = -EIO;
593 if (cqp_request->compl_info.maj_err_code == 0xFFFF) {
594 if (cqp_request->compl_info.min_err_code == 0x8002)
595 err_code = -EBUSY;
596 else if (cqp_request->compl_info.min_err_code == 0x8029) {
597 if (!rf->reset) {
598 rf->reset = true;
599 rf->gen_ops.request_reset(rf);
600 }
601 }
602 }
603 }
604
605 return err_code;
606 }
607
608 static const char *const irdma_cqp_cmd_names[IRDMA_MAX_CQP_OPS] = {
609 [IRDMA_OP_CEQ_DESTROY] = "Destroy CEQ Cmd",
610 [IRDMA_OP_AEQ_DESTROY] = "Destroy AEQ Cmd",
611 [IRDMA_OP_DELETE_ARP_CACHE_ENTRY] = "Delete ARP Cache Cmd",
612 [IRDMA_OP_MANAGE_APBVT_ENTRY] = "Manage APBV Table Entry Cmd",
613 [IRDMA_OP_CEQ_CREATE] = "CEQ Create Cmd",
614 [IRDMA_OP_AEQ_CREATE] = "AEQ Destroy Cmd",
615 [IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY] = "Manage Quad Hash Table Entry Cmd",
616 [IRDMA_OP_QP_MODIFY] = "Modify QP Cmd",
617 [IRDMA_OP_QP_UPLOAD_CONTEXT] = "Upload Context Cmd",
618 [IRDMA_OP_CQ_CREATE] = "Create CQ Cmd",
619 [IRDMA_OP_CQ_DESTROY] = "Destroy CQ Cmd",
620 [IRDMA_OP_QP_CREATE] = "Create QP Cmd",
621 [IRDMA_OP_QP_DESTROY] = "Destroy QP Cmd",
622 [IRDMA_OP_ALLOC_STAG] = "Allocate STag Cmd",
623 [IRDMA_OP_MR_REG_NON_SHARED] = "Register Non-Shared MR Cmd",
624 [IRDMA_OP_DEALLOC_STAG] = "Deallocate STag Cmd",
625 [IRDMA_OP_MW_ALLOC] = "Allocate Memory Window Cmd",
626 [IRDMA_OP_QP_FLUSH_WQES] = "Flush QP Cmd",
627 [IRDMA_OP_ADD_ARP_CACHE_ENTRY] = "Add ARP Cache Cmd",
628 [IRDMA_OP_MANAGE_PUSH_PAGE] = "Manage Push Page Cmd",
629 [IRDMA_OP_UPDATE_PE_SDS] = "Update PE SDs Cmd",
630 [IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE] = "Manage HMC PM Function Table Cmd",
631 [IRDMA_OP_SUSPEND] = "Suspend QP Cmd",
632 [IRDMA_OP_RESUME] = "Resume QP Cmd",
633 [IRDMA_OP_MANAGE_VF_PBLE_BP] = "Manage VF PBLE Backing Pages Cmd",
634 [IRDMA_OP_QUERY_FPM_VAL] = "Query FPM Values Cmd",
635 [IRDMA_OP_COMMIT_FPM_VAL] = "Commit FPM Values Cmd",
636 [IRDMA_OP_AH_CREATE] = "Create Address Handle Cmd",
637 [IRDMA_OP_AH_MODIFY] = "Modify Address Handle Cmd",
638 [IRDMA_OP_AH_DESTROY] = "Destroy Address Handle Cmd",
639 [IRDMA_OP_MC_CREATE] = "Create Multicast Group Cmd",
640 [IRDMA_OP_MC_DESTROY] = "Destroy Multicast Group Cmd",
641 [IRDMA_OP_MC_MODIFY] = "Modify Multicast Group Cmd",
642 [IRDMA_OP_STATS_ALLOCATE] = "Add Statistics Instance Cmd",
643 [IRDMA_OP_STATS_FREE] = "Free Statistics Instance Cmd",
644 [IRDMA_OP_STATS_GATHER] = "Gather Statistics Cmd",
645 [IRDMA_OP_WS_ADD_NODE] = "Add Work Scheduler Node Cmd",
646 [IRDMA_OP_WS_MODIFY_NODE] = "Modify Work Scheduler Node Cmd",
647 [IRDMA_OP_WS_DELETE_NODE] = "Delete Work Scheduler Node Cmd",
648 [IRDMA_OP_SET_UP_MAP] = "Set UP-UP Mapping Cmd",
649 [IRDMA_OP_GEN_AE] = "Generate AE Cmd",
650 [IRDMA_OP_QUERY_RDMA_FEATURES] = "RDMA Get Features Cmd",
651 [IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY] = "Allocate Local MAC Entry Cmd",
652 [IRDMA_OP_ADD_LOCAL_MAC_ENTRY] = "Add Local MAC Entry Cmd",
653 [IRDMA_OP_DELETE_LOCAL_MAC_ENTRY] = "Delete Local MAC Entry Cmd",
654 [IRDMA_OP_CQ_MODIFY] = "CQ Modify Cmd",
655 };
656
657 static const struct irdma_cqp_err_info irdma_noncrit_err_list[] = {
658 {0xffff, 0x8006, "Flush No Wqe Pending"},
659 {0xffff, 0x8007, "Modify QP Bad Close"},
660 {0xffff, 0x8009, "LLP Closed"},
661 {0xffff, 0x800a, "Reset Not Sent"}
662 };
663
664 /**
665 * irdma_cqp_crit_err - check if CQP error is critical
666 * @dev: pointer to dev structure
667 * @cqp_cmd: code for last CQP operation
668 * @maj_err_code: major error code
669 * @min_err_code: minot error code
670 */
irdma_cqp_crit_err(struct irdma_sc_dev * dev,u8 cqp_cmd,u16 maj_err_code,u16 min_err_code)671 bool irdma_cqp_crit_err(struct irdma_sc_dev *dev, u8 cqp_cmd,
672 u16 maj_err_code, u16 min_err_code)
673 {
674 int i;
675
676 for (i = 0; i < ARRAY_SIZE(irdma_noncrit_err_list); ++i) {
677 if (maj_err_code == irdma_noncrit_err_list[i].maj &&
678 min_err_code == irdma_noncrit_err_list[i].min) {
679 ibdev_dbg(to_ibdev(dev),
680 "CQP: [%s Error][%s] maj=0x%x min=0x%x\n",
681 irdma_noncrit_err_list[i].desc,
682 irdma_cqp_cmd_names[cqp_cmd], maj_err_code,
683 min_err_code);
684 return false;
685 }
686 }
687 return true;
688 }
689
690 /**
691 * irdma_handle_cqp_op - process cqp command
692 * @rf: RDMA PCI function
693 * @cqp_request: cqp request to process
694 */
irdma_handle_cqp_op(struct irdma_pci_f * rf,struct irdma_cqp_request * cqp_request)695 int irdma_handle_cqp_op(struct irdma_pci_f *rf,
696 struct irdma_cqp_request *cqp_request)
697 {
698 struct irdma_sc_dev *dev = &rf->sc_dev;
699 struct cqp_cmds_info *info = &cqp_request->info;
700 int status;
701 bool put_cqp_request = true;
702
703 if (rf->reset)
704 return -EBUSY;
705
706 irdma_get_cqp_request(cqp_request);
707 status = irdma_process_cqp_cmd(dev, info);
708 if (status)
709 goto err;
710
711 if (cqp_request->waiting) {
712 put_cqp_request = false;
713 status = irdma_wait_event(rf, cqp_request);
714 if (status)
715 goto err;
716 }
717
718 return 0;
719
720 err:
721 if (irdma_cqp_crit_err(dev, info->cqp_cmd,
722 cqp_request->compl_info.maj_err_code,
723 cqp_request->compl_info.min_err_code))
724 ibdev_err(&rf->iwdev->ibdev,
725 "[%s Error][op_code=%d] status=%d waiting=%d completion_err=%d maj=0x%x min=0x%x\n",
726 irdma_cqp_cmd_names[info->cqp_cmd], info->cqp_cmd, status, cqp_request->waiting,
727 cqp_request->compl_info.error, cqp_request->compl_info.maj_err_code,
728 cqp_request->compl_info.min_err_code);
729
730 if (put_cqp_request)
731 irdma_put_cqp_request(&rf->cqp, cqp_request);
732
733 return status;
734 }
735
irdma_qp_add_ref(struct ib_qp * ibqp)736 void irdma_qp_add_ref(struct ib_qp *ibqp)
737 {
738 struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;
739
740 refcount_inc(&iwqp->refcnt);
741 }
742
irdma_qp_rem_ref(struct ib_qp * ibqp)743 void irdma_qp_rem_ref(struct ib_qp *ibqp)
744 {
745 struct irdma_qp *iwqp = to_iwqp(ibqp);
746 struct irdma_device *iwdev = iwqp->iwdev;
747 u32 qp_num;
748 unsigned long flags;
749
750 spin_lock_irqsave(&iwdev->rf->qptable_lock, flags);
751 if (!refcount_dec_and_test(&iwqp->refcnt)) {
752 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
753 return;
754 }
755
756 qp_num = iwqp->ibqp.qp_num;
757 iwdev->rf->qp_table[qp_num] = NULL;
758 spin_unlock_irqrestore(&iwdev->rf->qptable_lock, flags);
759 complete(&iwqp->free_qp);
760 }
761
to_ibdev(struct irdma_sc_dev * dev)762 struct ib_device *to_ibdev(struct irdma_sc_dev *dev)
763 {
764 return &(container_of(dev, struct irdma_pci_f, sc_dev))->iwdev->ibdev;
765 }
766
767 /**
768 * irdma_get_qp - get qp address
769 * @device: iwarp device
770 * @qpn: qp number
771 */
irdma_get_qp(struct ib_device * device,int qpn)772 struct ib_qp *irdma_get_qp(struct ib_device *device, int qpn)
773 {
774 struct irdma_device *iwdev = to_iwdev(device);
775
776 if (qpn < IW_FIRST_QPN || qpn >= iwdev->rf->max_qp)
777 return NULL;
778
779 return &iwdev->rf->qp_table[qpn]->ibqp;
780 }
781
782 /**
783 * irdma_remove_cqp_head - return head entry and remove
784 * @dev: device
785 */
irdma_remove_cqp_head(struct irdma_sc_dev * dev)786 void *irdma_remove_cqp_head(struct irdma_sc_dev *dev)
787 {
788 struct list_head *entry;
789 struct list_head *list = &dev->cqp_cmd_head;
790
791 if (list_empty(list))
792 return NULL;
793
794 entry = list->next;
795 list_del(entry);
796
797 return entry;
798 }
799
800 /**
801 * irdma_cqp_sds_cmd - create cqp command for sd
802 * @dev: hardware control device structure
803 * @sdinfo: information for sd cqp
804 *
805 */
irdma_cqp_sds_cmd(struct irdma_sc_dev * dev,struct irdma_update_sds_info * sdinfo)806 int irdma_cqp_sds_cmd(struct irdma_sc_dev *dev,
807 struct irdma_update_sds_info *sdinfo)
808 {
809 struct irdma_cqp_request *cqp_request;
810 struct cqp_cmds_info *cqp_info;
811 struct irdma_pci_f *rf = dev_to_rf(dev);
812 int status;
813
814 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
815 if (!cqp_request)
816 return -ENOMEM;
817
818 cqp_info = &cqp_request->info;
819 memcpy(&cqp_info->in.u.update_pe_sds.info, sdinfo,
820 sizeof(cqp_info->in.u.update_pe_sds.info));
821 cqp_info->cqp_cmd = IRDMA_OP_UPDATE_PE_SDS;
822 cqp_info->post_sq = 1;
823 cqp_info->in.u.update_pe_sds.dev = dev;
824 cqp_info->in.u.update_pe_sds.scratch = (uintptr_t)cqp_request;
825
826 status = irdma_handle_cqp_op(rf, cqp_request);
827 irdma_put_cqp_request(&rf->cqp, cqp_request);
828
829 return status;
830 }
831
832 /**
833 * irdma_cqp_qp_suspend_resume - cqp command for suspend/resume
834 * @qp: hardware control qp
835 * @op: suspend or resume
836 */
irdma_cqp_qp_suspend_resume(struct irdma_sc_qp * qp,u8 op)837 int irdma_cqp_qp_suspend_resume(struct irdma_sc_qp *qp, u8 op)
838 {
839 struct irdma_sc_dev *dev = qp->dev;
840 struct irdma_cqp_request *cqp_request;
841 struct irdma_sc_cqp *cqp = dev->cqp;
842 struct cqp_cmds_info *cqp_info;
843 struct irdma_pci_f *rf = dev_to_rf(dev);
844 int status;
845
846 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
847 if (!cqp_request)
848 return -ENOMEM;
849
850 cqp_info = &cqp_request->info;
851 cqp_info->cqp_cmd = op;
852 cqp_info->in.u.suspend_resume.cqp = cqp;
853 cqp_info->in.u.suspend_resume.qp = qp;
854 cqp_info->in.u.suspend_resume.scratch = (uintptr_t)cqp_request;
855
856 status = irdma_handle_cqp_op(rf, cqp_request);
857 irdma_put_cqp_request(&rf->cqp, cqp_request);
858
859 return status;
860 }
861
862 /**
863 * irdma_term_modify_qp - modify qp for term message
864 * @qp: hardware control qp
865 * @next_state: qp's next state
866 * @term: terminate code
867 * @term_len: length
868 */
irdma_term_modify_qp(struct irdma_sc_qp * qp,u8 next_state,u8 term,u8 term_len)869 void irdma_term_modify_qp(struct irdma_sc_qp *qp, u8 next_state, u8 term,
870 u8 term_len)
871 {
872 struct irdma_qp *iwqp;
873
874 iwqp = qp->qp_uk.back_qp;
875 irdma_next_iw_state(iwqp, next_state, 0, term, term_len);
876 };
877
878 /**
879 * irdma_terminate_done - after terminate is completed
880 * @qp: hardware control qp
881 * @timeout_occurred: indicates if terminate timer expired
882 */
irdma_terminate_done(struct irdma_sc_qp * qp,int timeout_occurred)883 void irdma_terminate_done(struct irdma_sc_qp *qp, int timeout_occurred)
884 {
885 struct irdma_qp *iwqp;
886 u8 hte = 0;
887 bool first_time;
888 unsigned long flags;
889
890 iwqp = qp->qp_uk.back_qp;
891 spin_lock_irqsave(&iwqp->lock, flags);
892 if (iwqp->hte_added) {
893 iwqp->hte_added = 0;
894 hte = 1;
895 }
896 first_time = !(qp->term_flags & IRDMA_TERM_DONE);
897 qp->term_flags |= IRDMA_TERM_DONE;
898 spin_unlock_irqrestore(&iwqp->lock, flags);
899 if (first_time) {
900 if (!timeout_occurred)
901 irdma_terminate_del_timer(qp);
902
903 irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);
904 irdma_cm_disconn(iwqp);
905 }
906 }
907
irdma_terminate_timeout(struct timer_list * t)908 static void irdma_terminate_timeout(struct timer_list *t)
909 {
910 struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);
911 struct irdma_sc_qp *qp = &iwqp->sc_qp;
912
913 irdma_terminate_done(qp, 1);
914 irdma_qp_rem_ref(&iwqp->ibqp);
915 }
916
917 /**
918 * irdma_terminate_start_timer - start terminate timeout
919 * @qp: hardware control qp
920 */
irdma_terminate_start_timer(struct irdma_sc_qp * qp)921 void irdma_terminate_start_timer(struct irdma_sc_qp *qp)
922 {
923 struct irdma_qp *iwqp;
924
925 iwqp = qp->qp_uk.back_qp;
926 irdma_qp_add_ref(&iwqp->ibqp);
927 timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);
928 iwqp->terminate_timer.expires = jiffies + HZ;
929
930 add_timer(&iwqp->terminate_timer);
931 }
932
933 /**
934 * irdma_terminate_del_timer - delete terminate timeout
935 * @qp: hardware control qp
936 */
irdma_terminate_del_timer(struct irdma_sc_qp * qp)937 void irdma_terminate_del_timer(struct irdma_sc_qp *qp)
938 {
939 struct irdma_qp *iwqp;
940 int ret;
941
942 iwqp = qp->qp_uk.back_qp;
943 ret = del_timer(&iwqp->terminate_timer);
944 if (ret)
945 irdma_qp_rem_ref(&iwqp->ibqp);
946 }
947
948 /**
949 * irdma_cqp_query_fpm_val_cmd - send cqp command for fpm
950 * @dev: function device struct
951 * @val_mem: buffer for fpm
952 * @hmc_fn_id: function id for fpm
953 */
irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u8 hmc_fn_id)954 int irdma_cqp_query_fpm_val_cmd(struct irdma_sc_dev *dev,
955 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
956 {
957 struct irdma_cqp_request *cqp_request;
958 struct cqp_cmds_info *cqp_info;
959 struct irdma_pci_f *rf = dev_to_rf(dev);
960 int status;
961
962 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
963 if (!cqp_request)
964 return -ENOMEM;
965
966 cqp_info = &cqp_request->info;
967 cqp_request->param = NULL;
968 cqp_info->in.u.query_fpm_val.cqp = dev->cqp;
969 cqp_info->in.u.query_fpm_val.fpm_val_pa = val_mem->pa;
970 cqp_info->in.u.query_fpm_val.fpm_val_va = val_mem->va;
971 cqp_info->in.u.query_fpm_val.hmc_fn_id = hmc_fn_id;
972 cqp_info->cqp_cmd = IRDMA_OP_QUERY_FPM_VAL;
973 cqp_info->post_sq = 1;
974 cqp_info->in.u.query_fpm_val.scratch = (uintptr_t)cqp_request;
975
976 status = irdma_handle_cqp_op(rf, cqp_request);
977 irdma_put_cqp_request(&rf->cqp, cqp_request);
978
979 return status;
980 }
981
982 /**
983 * irdma_cqp_commit_fpm_val_cmd - commit fpm values in hw
984 * @dev: hardware control device structure
985 * @val_mem: buffer with fpm values
986 * @hmc_fn_id: function id for fpm
987 */
irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev * dev,struct irdma_dma_mem * val_mem,u8 hmc_fn_id)988 int irdma_cqp_commit_fpm_val_cmd(struct irdma_sc_dev *dev,
989 struct irdma_dma_mem *val_mem, u8 hmc_fn_id)
990 {
991 struct irdma_cqp_request *cqp_request;
992 struct cqp_cmds_info *cqp_info;
993 struct irdma_pci_f *rf = dev_to_rf(dev);
994 int status;
995
996 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
997 if (!cqp_request)
998 return -ENOMEM;
999
1000 cqp_info = &cqp_request->info;
1001 cqp_request->param = NULL;
1002 cqp_info->in.u.commit_fpm_val.cqp = dev->cqp;
1003 cqp_info->in.u.commit_fpm_val.fpm_val_pa = val_mem->pa;
1004 cqp_info->in.u.commit_fpm_val.fpm_val_va = val_mem->va;
1005 cqp_info->in.u.commit_fpm_val.hmc_fn_id = hmc_fn_id;
1006 cqp_info->cqp_cmd = IRDMA_OP_COMMIT_FPM_VAL;
1007 cqp_info->post_sq = 1;
1008 cqp_info->in.u.commit_fpm_val.scratch = (uintptr_t)cqp_request;
1009
1010 status = irdma_handle_cqp_op(rf, cqp_request);
1011 irdma_put_cqp_request(&rf->cqp, cqp_request);
1012
1013 return status;
1014 }
1015
1016 /**
1017 * irdma_cqp_cq_create_cmd - create a cq for the cqp
1018 * @dev: device pointer
1019 * @cq: pointer to created cq
1020 */
irdma_cqp_cq_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1021 int irdma_cqp_cq_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1022 {
1023 struct irdma_pci_f *rf = dev_to_rf(dev);
1024 struct irdma_cqp *iwcqp = &rf->cqp;
1025 struct irdma_cqp_request *cqp_request;
1026 struct cqp_cmds_info *cqp_info;
1027 int status;
1028
1029 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1030 if (!cqp_request)
1031 return -ENOMEM;
1032
1033 cqp_info = &cqp_request->info;
1034 cqp_info->cqp_cmd = IRDMA_OP_CQ_CREATE;
1035 cqp_info->post_sq = 1;
1036 cqp_info->in.u.cq_create.cq = cq;
1037 cqp_info->in.u.cq_create.scratch = (uintptr_t)cqp_request;
1038
1039 status = irdma_handle_cqp_op(rf, cqp_request);
1040 irdma_put_cqp_request(iwcqp, cqp_request);
1041
1042 return status;
1043 }
1044
1045 /**
1046 * irdma_cqp_qp_create_cmd - create a qp for the cqp
1047 * @dev: device pointer
1048 * @qp: pointer to created qp
1049 */
irdma_cqp_qp_create_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1050 int irdma_cqp_qp_create_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1051 {
1052 struct irdma_pci_f *rf = dev_to_rf(dev);
1053 struct irdma_cqp *iwcqp = &rf->cqp;
1054 struct irdma_cqp_request *cqp_request;
1055 struct cqp_cmds_info *cqp_info;
1056 struct irdma_create_qp_info *qp_info;
1057 int status;
1058
1059 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1060 if (!cqp_request)
1061 return -ENOMEM;
1062
1063 cqp_info = &cqp_request->info;
1064 qp_info = &cqp_request->info.in.u.qp_create.info;
1065 memset(qp_info, 0, sizeof(*qp_info));
1066 qp_info->cq_num_valid = true;
1067 qp_info->next_iwarp_state = IRDMA_QP_STATE_RTS;
1068 cqp_info->cqp_cmd = IRDMA_OP_QP_CREATE;
1069 cqp_info->post_sq = 1;
1070 cqp_info->in.u.qp_create.qp = qp;
1071 cqp_info->in.u.qp_create.scratch = (uintptr_t)cqp_request;
1072
1073 status = irdma_handle_cqp_op(rf, cqp_request);
1074 irdma_put_cqp_request(iwcqp, cqp_request);
1075
1076 return status;
1077 }
1078
1079 /**
1080 * irdma_dealloc_push_page - free a push page for qp
1081 * @rf: RDMA PCI function
1082 * @qp: hardware control qp
1083 */
irdma_dealloc_push_page(struct irdma_pci_f * rf,struct irdma_sc_qp * qp)1084 static void irdma_dealloc_push_page(struct irdma_pci_f *rf,
1085 struct irdma_sc_qp *qp)
1086 {
1087 struct irdma_cqp_request *cqp_request;
1088 struct cqp_cmds_info *cqp_info;
1089 int status;
1090
1091 if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX)
1092 return;
1093
1094 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, false);
1095 if (!cqp_request)
1096 return;
1097
1098 cqp_info = &cqp_request->info;
1099 cqp_info->cqp_cmd = IRDMA_OP_MANAGE_PUSH_PAGE;
1100 cqp_info->post_sq = 1;
1101 cqp_info->in.u.manage_push_page.info.push_idx = qp->push_idx;
1102 cqp_info->in.u.manage_push_page.info.qs_handle = qp->qs_handle;
1103 cqp_info->in.u.manage_push_page.info.free_page = 1;
1104 cqp_info->in.u.manage_push_page.info.push_page_type = 0;
1105 cqp_info->in.u.manage_push_page.cqp = &rf->cqp.sc_cqp;
1106 cqp_info->in.u.manage_push_page.scratch = (uintptr_t)cqp_request;
1107 status = irdma_handle_cqp_op(rf, cqp_request);
1108 if (!status)
1109 qp->push_idx = IRDMA_INVALID_PUSH_PAGE_INDEX;
1110 irdma_put_cqp_request(&rf->cqp, cqp_request);
1111 }
1112
1113 /**
1114 * irdma_free_qp_rsrc - free up memory resources for qp
1115 * @iwqp: qp ptr (user or kernel)
1116 */
irdma_free_qp_rsrc(struct irdma_qp * iwqp)1117 void irdma_free_qp_rsrc(struct irdma_qp *iwqp)
1118 {
1119 struct irdma_device *iwdev = iwqp->iwdev;
1120 struct irdma_pci_f *rf = iwdev->rf;
1121 u32 qp_num = iwqp->ibqp.qp_num;
1122
1123 irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);
1124 irdma_dealloc_push_page(rf, &iwqp->sc_qp);
1125 if (iwqp->sc_qp.vsi) {
1126 irdma_qp_rem_qos(&iwqp->sc_qp);
1127 iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,
1128 iwqp->sc_qp.user_pri);
1129 }
1130
1131 if (qp_num > 2)
1132 irdma_free_rsrc(rf, rf->allocated_qps, qp_num);
1133 dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,
1134 iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);
1135 iwqp->q2_ctx_mem.va = NULL;
1136 dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,
1137 iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);
1138 iwqp->kqp.dma_mem.va = NULL;
1139 kfree(iwqp->kqp.sq_wrid_mem);
1140 kfree(iwqp->kqp.rq_wrid_mem);
1141 }
1142
1143 /**
1144 * irdma_cq_wq_destroy - send cq destroy cqp
1145 * @rf: RDMA PCI function
1146 * @cq: hardware control cq
1147 */
irdma_cq_wq_destroy(struct irdma_pci_f * rf,struct irdma_sc_cq * cq)1148 void irdma_cq_wq_destroy(struct irdma_pci_f *rf, struct irdma_sc_cq *cq)
1149 {
1150 struct irdma_cqp_request *cqp_request;
1151 struct cqp_cmds_info *cqp_info;
1152
1153 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1154 if (!cqp_request)
1155 return;
1156
1157 cqp_info = &cqp_request->info;
1158 cqp_info->cqp_cmd = IRDMA_OP_CQ_DESTROY;
1159 cqp_info->post_sq = 1;
1160 cqp_info->in.u.cq_destroy.cq = cq;
1161 cqp_info->in.u.cq_destroy.scratch = (uintptr_t)cqp_request;
1162
1163 irdma_handle_cqp_op(rf, cqp_request);
1164 irdma_put_cqp_request(&rf->cqp, cqp_request);
1165 }
1166
1167 /**
1168 * irdma_hw_modify_qp_callback - handle state for modifyQPs that don't wait
1169 * @cqp_request: modify QP completion
1170 */
irdma_hw_modify_qp_callback(struct irdma_cqp_request * cqp_request)1171 static void irdma_hw_modify_qp_callback(struct irdma_cqp_request *cqp_request)
1172 {
1173 struct cqp_cmds_info *cqp_info;
1174 struct irdma_qp *iwqp;
1175
1176 cqp_info = &cqp_request->info;
1177 iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;
1178 atomic_dec(&iwqp->hw_mod_qp_pend);
1179 wake_up(&iwqp->mod_qp_waitq);
1180 }
1181
1182 /**
1183 * irdma_hw_modify_qp - setup cqp for modify qp
1184 * @iwdev: RDMA device
1185 * @iwqp: qp ptr (user or kernel)
1186 * @info: info for modify qp
1187 * @wait: flag to wait or not for modify qp completion
1188 */
irdma_hw_modify_qp(struct irdma_device * iwdev,struct irdma_qp * iwqp,struct irdma_modify_qp_info * info,bool wait)1189 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,
1190 struct irdma_modify_qp_info *info, bool wait)
1191 {
1192 int status;
1193 struct irdma_pci_f *rf = iwdev->rf;
1194 struct irdma_cqp_request *cqp_request;
1195 struct cqp_cmds_info *cqp_info;
1196 struct irdma_modify_qp_info *m_info;
1197
1198 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
1199 if (!cqp_request)
1200 return -ENOMEM;
1201
1202 if (!wait) {
1203 cqp_request->callback_fcn = irdma_hw_modify_qp_callback;
1204 atomic_inc(&iwqp->hw_mod_qp_pend);
1205 }
1206 cqp_info = &cqp_request->info;
1207 m_info = &cqp_info->in.u.qp_modify.info;
1208 memcpy(m_info, info, sizeof(*m_info));
1209 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1210 cqp_info->post_sq = 1;
1211 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1212 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1213 status = irdma_handle_cqp_op(rf, cqp_request);
1214 irdma_put_cqp_request(&rf->cqp, cqp_request);
1215 if (status) {
1216 if (rdma_protocol_roce(&iwdev->ibdev, 1))
1217 return status;
1218
1219 switch (m_info->next_iwarp_state) {
1220 struct irdma_gen_ae_info ae_info;
1221
1222 case IRDMA_QP_STATE_RTS:
1223 case IRDMA_QP_STATE_IDLE:
1224 case IRDMA_QP_STATE_TERMINATE:
1225 case IRDMA_QP_STATE_CLOSING:
1226 if (info->curr_iwarp_state == IRDMA_QP_STATE_IDLE)
1227 irdma_send_reset(iwqp->cm_node);
1228 else
1229 iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;
1230 if (!wait) {
1231 ae_info.ae_code = IRDMA_AE_BAD_CLOSE;
1232 ae_info.ae_src = 0;
1233 irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);
1234 } else {
1235 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp,
1236 wait);
1237 if (!cqp_request)
1238 return -ENOMEM;
1239
1240 cqp_info = &cqp_request->info;
1241 m_info = &cqp_info->in.u.qp_modify.info;
1242 memcpy(m_info, info, sizeof(*m_info));
1243 cqp_info->cqp_cmd = IRDMA_OP_QP_MODIFY;
1244 cqp_info->post_sq = 1;
1245 cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;
1246 cqp_info->in.u.qp_modify.scratch = (uintptr_t)cqp_request;
1247 m_info->next_iwarp_state = IRDMA_QP_STATE_ERROR;
1248 m_info->reset_tcp_conn = true;
1249 irdma_handle_cqp_op(rf, cqp_request);
1250 irdma_put_cqp_request(&rf->cqp, cqp_request);
1251 }
1252 break;
1253 case IRDMA_QP_STATE_ERROR:
1254 default:
1255 break;
1256 }
1257 }
1258
1259 return status;
1260 }
1261
1262 /**
1263 * irdma_cqp_cq_destroy_cmd - destroy the cqp cq
1264 * @dev: device pointer
1265 * @cq: pointer to cq
1266 */
irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_cq * cq)1267 void irdma_cqp_cq_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_cq *cq)
1268 {
1269 struct irdma_pci_f *rf = dev_to_rf(dev);
1270
1271 irdma_cq_wq_destroy(rf, cq);
1272 }
1273
1274 /**
1275 * irdma_cqp_qp_destroy_cmd - destroy the cqp
1276 * @dev: device pointer
1277 * @qp: pointer to qp
1278 */
irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1279 int irdma_cqp_qp_destroy_cmd(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1280 {
1281 struct irdma_pci_f *rf = dev_to_rf(dev);
1282 struct irdma_cqp *iwcqp = &rf->cqp;
1283 struct irdma_cqp_request *cqp_request;
1284 struct cqp_cmds_info *cqp_info;
1285 int status;
1286
1287 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, true);
1288 if (!cqp_request)
1289 return -ENOMEM;
1290
1291 cqp_info = &cqp_request->info;
1292 memset(cqp_info, 0, sizeof(*cqp_info));
1293 cqp_info->cqp_cmd = IRDMA_OP_QP_DESTROY;
1294 cqp_info->post_sq = 1;
1295 cqp_info->in.u.qp_destroy.qp = qp;
1296 cqp_info->in.u.qp_destroy.scratch = (uintptr_t)cqp_request;
1297 cqp_info->in.u.qp_destroy.remove_hash_idx = true;
1298
1299 status = irdma_handle_cqp_op(rf, cqp_request);
1300 irdma_put_cqp_request(&rf->cqp, cqp_request);
1301
1302 return status;
1303 }
1304
1305 /**
1306 * irdma_ieq_mpa_crc_ae - generate AE for crc error
1307 * @dev: hardware control device structure
1308 * @qp: hardware control qp
1309 */
irdma_ieq_mpa_crc_ae(struct irdma_sc_dev * dev,struct irdma_sc_qp * qp)1310 void irdma_ieq_mpa_crc_ae(struct irdma_sc_dev *dev, struct irdma_sc_qp *qp)
1311 {
1312 struct irdma_gen_ae_info info = {};
1313 struct irdma_pci_f *rf = dev_to_rf(dev);
1314
1315 ibdev_dbg(&rf->iwdev->ibdev, "AEQ: Generate MPA CRC AE\n");
1316 info.ae_code = IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR;
1317 info.ae_src = IRDMA_AE_SOURCE_RQ;
1318 irdma_gen_ae(rf, qp, &info, false);
1319 }
1320
1321 /**
1322 * irdma_init_hash_desc - initialize hash for crc calculation
1323 * @desc: cryption type
1324 */
irdma_init_hash_desc(struct shash_desc ** desc)1325 int irdma_init_hash_desc(struct shash_desc **desc)
1326 {
1327 struct crypto_shash *tfm;
1328 struct shash_desc *tdesc;
1329
1330 tfm = crypto_alloc_shash("crc32c", 0, 0);
1331 if (IS_ERR(tfm))
1332 return -EINVAL;
1333
1334 tdesc = kzalloc(sizeof(*tdesc) + crypto_shash_descsize(tfm),
1335 GFP_KERNEL);
1336 if (!tdesc) {
1337 crypto_free_shash(tfm);
1338 return -EINVAL;
1339 }
1340
1341 tdesc->tfm = tfm;
1342 *desc = tdesc;
1343
1344 return 0;
1345 }
1346
1347 /**
1348 * irdma_free_hash_desc - free hash desc
1349 * @desc: to be freed
1350 */
irdma_free_hash_desc(struct shash_desc * desc)1351 void irdma_free_hash_desc(struct shash_desc *desc)
1352 {
1353 if (desc) {
1354 crypto_free_shash(desc->tfm);
1355 kfree(desc);
1356 }
1357 }
1358
1359 /**
1360 * irdma_ieq_check_mpacrc - check if mpa crc is OK
1361 * @desc: desc for hash
1362 * @addr: address of buffer for crc
1363 * @len: length of buffer
1364 * @val: value to be compared
1365 */
irdma_ieq_check_mpacrc(struct shash_desc * desc,void * addr,u32 len,u32 val)1366 int irdma_ieq_check_mpacrc(struct shash_desc *desc, void *addr, u32 len,
1367 u32 val)
1368 {
1369 u32 crc = 0;
1370 int ret;
1371 int ret_code = 0;
1372
1373 crypto_shash_init(desc);
1374 ret = crypto_shash_update(desc, addr, len);
1375 if (!ret)
1376 crypto_shash_final(desc, (u8 *)&crc);
1377 if (crc != val)
1378 ret_code = -EINVAL;
1379
1380 return ret_code;
1381 }
1382
1383 /**
1384 * irdma_ieq_get_qp - get qp based on quad in puda buffer
1385 * @dev: hardware control device structure
1386 * @buf: receive puda buffer on exception q
1387 */
irdma_ieq_get_qp(struct irdma_sc_dev * dev,struct irdma_puda_buf * buf)1388 struct irdma_sc_qp *irdma_ieq_get_qp(struct irdma_sc_dev *dev,
1389 struct irdma_puda_buf *buf)
1390 {
1391 struct irdma_qp *iwqp;
1392 struct irdma_cm_node *cm_node;
1393 struct irdma_device *iwdev = buf->vsi->back_vsi;
1394 u32 loc_addr[4] = {};
1395 u32 rem_addr[4] = {};
1396 u16 loc_port, rem_port;
1397 struct ipv6hdr *ip6h;
1398 struct iphdr *iph = (struct iphdr *)buf->iph;
1399 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1400
1401 if (iph->version == 4) {
1402 loc_addr[0] = ntohl(iph->daddr);
1403 rem_addr[0] = ntohl(iph->saddr);
1404 } else {
1405 ip6h = (struct ipv6hdr *)buf->iph;
1406 irdma_copy_ip_ntohl(loc_addr, ip6h->daddr.in6_u.u6_addr32);
1407 irdma_copy_ip_ntohl(rem_addr, ip6h->saddr.in6_u.u6_addr32);
1408 }
1409 loc_port = ntohs(tcph->dest);
1410 rem_port = ntohs(tcph->source);
1411 cm_node = irdma_find_node(&iwdev->cm_core, rem_port, rem_addr, loc_port,
1412 loc_addr, buf->vlan_valid ? buf->vlan_id : 0xFFFF);
1413 if (!cm_node)
1414 return NULL;
1415
1416 iwqp = cm_node->iwqp;
1417 irdma_rem_ref_cm_node(cm_node);
1418
1419 return &iwqp->sc_qp;
1420 }
1421
1422 /**
1423 * irdma_send_ieq_ack - ACKs for duplicate or OOO partials FPDUs
1424 * @qp: qp ptr
1425 */
irdma_send_ieq_ack(struct irdma_sc_qp * qp)1426 void irdma_send_ieq_ack(struct irdma_sc_qp *qp)
1427 {
1428 struct irdma_cm_node *cm_node = ((struct irdma_qp *)qp->qp_uk.back_qp)->cm_node;
1429 struct irdma_puda_buf *buf = qp->pfpdu.lastrcv_buf;
1430 struct tcphdr *tcph = (struct tcphdr *)buf->tcph;
1431
1432 cm_node->tcp_cntxt.rcv_nxt = qp->pfpdu.nextseqnum;
1433 cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq);
1434
1435 irdma_send_ack(cm_node);
1436 }
1437
1438 /**
1439 * irdma_puda_ieq_get_ah_info - get AH info from IEQ buffer
1440 * @qp: qp pointer
1441 * @ah_info: AH info pointer
1442 */
irdma_puda_ieq_get_ah_info(struct irdma_sc_qp * qp,struct irdma_ah_info * ah_info)1443 void irdma_puda_ieq_get_ah_info(struct irdma_sc_qp *qp,
1444 struct irdma_ah_info *ah_info)
1445 {
1446 struct irdma_puda_buf *buf = qp->pfpdu.ah_buf;
1447 struct iphdr *iph;
1448 struct ipv6hdr *ip6h;
1449
1450 memset(ah_info, 0, sizeof(*ah_info));
1451 ah_info->do_lpbk = true;
1452 ah_info->vlan_tag = buf->vlan_id;
1453 ah_info->insert_vlan_tag = buf->vlan_valid;
1454 ah_info->ipv4_valid = buf->ipv4;
1455 ah_info->vsi = qp->vsi;
1456
1457 if (buf->smac_valid)
1458 ether_addr_copy(ah_info->mac_addr, buf->smac);
1459
1460 if (buf->ipv4) {
1461 ah_info->ipv4_valid = true;
1462 iph = (struct iphdr *)buf->iph;
1463 ah_info->hop_ttl = iph->ttl;
1464 ah_info->tc_tos = iph->tos;
1465 ah_info->dest_ip_addr[0] = ntohl(iph->daddr);
1466 ah_info->src_ip_addr[0] = ntohl(iph->saddr);
1467 } else {
1468 ip6h = (struct ipv6hdr *)buf->iph;
1469 ah_info->hop_ttl = ip6h->hop_limit;
1470 ah_info->tc_tos = ip6h->priority;
1471 irdma_copy_ip_ntohl(ah_info->dest_ip_addr,
1472 ip6h->daddr.in6_u.u6_addr32);
1473 irdma_copy_ip_ntohl(ah_info->src_ip_addr,
1474 ip6h->saddr.in6_u.u6_addr32);
1475 }
1476
1477 ah_info->dst_arpindex = irdma_arp_table(dev_to_rf(qp->dev),
1478 ah_info->dest_ip_addr,
1479 ah_info->ipv4_valid,
1480 NULL, IRDMA_ARP_RESOLVE);
1481 }
1482
1483 /**
1484 * irdma_gen1_ieq_update_tcpip_info - update tcpip in the buffer
1485 * @buf: puda to update
1486 * @len: length of buffer
1487 * @seqnum: seq number for tcp
1488 */
irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1489 static void irdma_gen1_ieq_update_tcpip_info(struct irdma_puda_buf *buf,
1490 u16 len, u32 seqnum)
1491 {
1492 struct tcphdr *tcph;
1493 struct iphdr *iph;
1494 u16 iphlen;
1495 u16 pktsize;
1496 u8 *addr = buf->mem.va;
1497
1498 iphlen = (buf->ipv4) ? 20 : 40;
1499 iph = (struct iphdr *)(addr + buf->maclen);
1500 tcph = (struct tcphdr *)(addr + buf->maclen + iphlen);
1501 pktsize = len + buf->tcphlen + iphlen;
1502 iph->tot_len = htons(pktsize);
1503 tcph->seq = htonl(seqnum);
1504 }
1505
1506 /**
1507 * irdma_ieq_update_tcpip_info - update tcpip in the buffer
1508 * @buf: puda to update
1509 * @len: length of buffer
1510 * @seqnum: seq number for tcp
1511 */
irdma_ieq_update_tcpip_info(struct irdma_puda_buf * buf,u16 len,u32 seqnum)1512 void irdma_ieq_update_tcpip_info(struct irdma_puda_buf *buf, u16 len,
1513 u32 seqnum)
1514 {
1515 struct tcphdr *tcph;
1516 u8 *addr;
1517
1518 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1519 return irdma_gen1_ieq_update_tcpip_info(buf, len, seqnum);
1520
1521 addr = buf->mem.va;
1522 tcph = (struct tcphdr *)addr;
1523 tcph->seq = htonl(seqnum);
1524 }
1525
1526 /**
1527 * irdma_gen1_puda_get_tcpip_info - get tcpip info from puda
1528 * buffer
1529 * @info: to get information
1530 * @buf: puda buffer
1531 */
irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1532 static int irdma_gen1_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1533 struct irdma_puda_buf *buf)
1534 {
1535 struct iphdr *iph;
1536 struct ipv6hdr *ip6h;
1537 struct tcphdr *tcph;
1538 u16 iphlen;
1539 u16 pkt_len;
1540 u8 *mem = buf->mem.va;
1541 struct ethhdr *ethh = buf->mem.va;
1542
1543 if (ethh->h_proto == htons(0x8100)) {
1544 info->vlan_valid = true;
1545 buf->vlan_id = ntohs(((struct vlan_ethhdr *)ethh)->h_vlan_TCI) &
1546 VLAN_VID_MASK;
1547 }
1548
1549 buf->maclen = (info->vlan_valid) ? 18 : 14;
1550 iphlen = (info->l3proto) ? 40 : 20;
1551 buf->ipv4 = (info->l3proto) ? false : true;
1552 buf->iph = mem + buf->maclen;
1553 iph = (struct iphdr *)buf->iph;
1554 buf->tcph = buf->iph + iphlen;
1555 tcph = (struct tcphdr *)buf->tcph;
1556
1557 if (buf->ipv4) {
1558 pkt_len = ntohs(iph->tot_len);
1559 } else {
1560 ip6h = (struct ipv6hdr *)buf->iph;
1561 pkt_len = ntohs(ip6h->payload_len) + iphlen;
1562 }
1563
1564 buf->totallen = pkt_len + buf->maclen;
1565
1566 if (info->payload_len < buf->totallen) {
1567 ibdev_dbg(to_ibdev(buf->vsi->dev),
1568 "ERR: payload_len = 0x%x totallen expected0x%x\n",
1569 info->payload_len, buf->totallen);
1570 return -EINVAL;
1571 }
1572
1573 buf->tcphlen = tcph->doff << 2;
1574 buf->datalen = pkt_len - iphlen - buf->tcphlen;
1575 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1576 buf->hdrlen = buf->maclen + iphlen + buf->tcphlen;
1577 buf->seqnum = ntohl(tcph->seq);
1578
1579 return 0;
1580 }
1581
1582 /**
1583 * irdma_puda_get_tcpip_info - get tcpip info from puda buffer
1584 * @info: to get information
1585 * @buf: puda buffer
1586 */
irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info * info,struct irdma_puda_buf * buf)1587 int irdma_puda_get_tcpip_info(struct irdma_puda_cmpl_info *info,
1588 struct irdma_puda_buf *buf)
1589 {
1590 struct tcphdr *tcph;
1591 u32 pkt_len;
1592 u8 *mem;
1593
1594 if (buf->vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1595 return irdma_gen1_puda_get_tcpip_info(info, buf);
1596
1597 mem = buf->mem.va;
1598 buf->vlan_valid = info->vlan_valid;
1599 if (info->vlan_valid)
1600 buf->vlan_id = info->vlan;
1601
1602 buf->ipv4 = info->ipv4;
1603 if (buf->ipv4)
1604 buf->iph = mem + IRDMA_IPV4_PAD;
1605 else
1606 buf->iph = mem;
1607
1608 buf->tcph = mem + IRDMA_TCP_OFFSET;
1609 tcph = (struct tcphdr *)buf->tcph;
1610 pkt_len = info->payload_len;
1611 buf->totallen = pkt_len;
1612 buf->tcphlen = tcph->doff << 2;
1613 buf->datalen = pkt_len - IRDMA_TCP_OFFSET - buf->tcphlen;
1614 buf->data = buf->datalen ? buf->tcph + buf->tcphlen : NULL;
1615 buf->hdrlen = IRDMA_TCP_OFFSET + buf->tcphlen;
1616 buf->seqnum = ntohl(tcph->seq);
1617
1618 if (info->smac_valid) {
1619 ether_addr_copy(buf->smac, info->smac);
1620 buf->smac_valid = true;
1621 }
1622
1623 return 0;
1624 }
1625
1626 /**
1627 * irdma_hw_stats_timeout - Stats timer-handler which updates all HW stats
1628 * @t: timer_list pointer
1629 */
irdma_hw_stats_timeout(struct timer_list * t)1630 static void irdma_hw_stats_timeout(struct timer_list *t)
1631 {
1632 struct irdma_vsi_pestat *pf_devstat =
1633 from_timer(pf_devstat, t, stats_timer);
1634 struct irdma_sc_vsi *sc_vsi = pf_devstat->vsi;
1635
1636 if (sc_vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1637 irdma_cqp_gather_stats_gen1(sc_vsi->dev, sc_vsi->pestat);
1638 else
1639 irdma_cqp_gather_stats_cmd(sc_vsi->dev, sc_vsi->pestat, false);
1640
1641 mod_timer(&pf_devstat->stats_timer,
1642 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1643 }
1644
1645 /**
1646 * irdma_hw_stats_start_timer - Start periodic stats timer
1647 * @vsi: vsi structure pointer
1648 */
irdma_hw_stats_start_timer(struct irdma_sc_vsi * vsi)1649 void irdma_hw_stats_start_timer(struct irdma_sc_vsi *vsi)
1650 {
1651 struct irdma_vsi_pestat *devstat = vsi->pestat;
1652
1653 timer_setup(&devstat->stats_timer, irdma_hw_stats_timeout, 0);
1654 mod_timer(&devstat->stats_timer,
1655 jiffies + msecs_to_jiffies(STATS_TIMER_DELAY));
1656 }
1657
1658 /**
1659 * irdma_hw_stats_stop_timer - Delete periodic stats timer
1660 * @vsi: pointer to vsi structure
1661 */
irdma_hw_stats_stop_timer(struct irdma_sc_vsi * vsi)1662 void irdma_hw_stats_stop_timer(struct irdma_sc_vsi *vsi)
1663 {
1664 struct irdma_vsi_pestat *devstat = vsi->pestat;
1665
1666 del_timer_sync(&devstat->stats_timer);
1667 }
1668
1669 /**
1670 * irdma_process_stats - Checking for wrap and update stats
1671 * @pestat: stats structure pointer
1672 */
irdma_process_stats(struct irdma_vsi_pestat * pestat)1673 static inline void irdma_process_stats(struct irdma_vsi_pestat *pestat)
1674 {
1675 sc_vsi_update_stats(pestat->vsi);
1676 }
1677
1678 /**
1679 * irdma_cqp_gather_stats_gen1 - Gather stats
1680 * @dev: pointer to device structure
1681 * @pestat: statistics structure
1682 */
irdma_cqp_gather_stats_gen1(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat)1683 void irdma_cqp_gather_stats_gen1(struct irdma_sc_dev *dev,
1684 struct irdma_vsi_pestat *pestat)
1685 {
1686 struct irdma_gather_stats *gather_stats =
1687 pestat->gather_info.gather_stats_va;
1688 u32 stats_inst_offset_32;
1689 u32 stats_inst_offset_64;
1690
1691 stats_inst_offset_32 = (pestat->gather_info.use_stats_inst) ?
1692 pestat->gather_info.stats_inst_index :
1693 pestat->hw->hmc.hmc_fn_id;
1694 stats_inst_offset_32 *= 4;
1695 stats_inst_offset_64 = stats_inst_offset_32 * 2;
1696
1697 gather_stats->rxvlanerr =
1698 rd32(dev->hw,
1699 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_RXVLANERR]
1700 + stats_inst_offset_32);
1701 gather_stats->ip4rxdiscard =
1702 rd32(dev->hw,
1703 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXDISCARD]
1704 + stats_inst_offset_32);
1705 gather_stats->ip4rxtrunc =
1706 rd32(dev->hw,
1707 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4RXTRUNC]
1708 + stats_inst_offset_32);
1709 gather_stats->ip4txnoroute =
1710 rd32(dev->hw,
1711 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP4TXNOROUTE]
1712 + stats_inst_offset_32);
1713 gather_stats->ip6rxdiscard =
1714 rd32(dev->hw,
1715 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXDISCARD]
1716 + stats_inst_offset_32);
1717 gather_stats->ip6rxtrunc =
1718 rd32(dev->hw,
1719 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6RXTRUNC]
1720 + stats_inst_offset_32);
1721 gather_stats->ip6txnoroute =
1722 rd32(dev->hw,
1723 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_IP6TXNOROUTE]
1724 + stats_inst_offset_32);
1725 gather_stats->tcprtxseg =
1726 rd32(dev->hw,
1727 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRTXSEG]
1728 + stats_inst_offset_32);
1729 gather_stats->tcprxopterr =
1730 rd32(dev->hw,
1731 dev->hw_stats_regs_32[IRDMA_HW_STAT_INDEX_TCPRXOPTERR]
1732 + stats_inst_offset_32);
1733
1734 gather_stats->ip4rxocts =
1735 rd64(dev->hw,
1736 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXOCTS]
1737 + stats_inst_offset_64);
1738 gather_stats->ip4rxpkts =
1739 rd64(dev->hw,
1740 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXPKTS]
1741 + stats_inst_offset_64);
1742 gather_stats->ip4txfrag =
1743 rd64(dev->hw,
1744 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXFRAGS]
1745 + stats_inst_offset_64);
1746 gather_stats->ip4rxmcpkts =
1747 rd64(dev->hw,
1748 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4RXMCPKTS]
1749 + stats_inst_offset_64);
1750 gather_stats->ip4txocts =
1751 rd64(dev->hw,
1752 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXOCTS]
1753 + stats_inst_offset_64);
1754 gather_stats->ip4txpkts =
1755 rd64(dev->hw,
1756 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXPKTS]
1757 + stats_inst_offset_64);
1758 gather_stats->ip4txfrag =
1759 rd64(dev->hw,
1760 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXFRAGS]
1761 + stats_inst_offset_64);
1762 gather_stats->ip4txmcpkts =
1763 rd64(dev->hw,
1764 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP4TXMCPKTS]
1765 + stats_inst_offset_64);
1766 gather_stats->ip6rxocts =
1767 rd64(dev->hw,
1768 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXOCTS]
1769 + stats_inst_offset_64);
1770 gather_stats->ip6rxpkts =
1771 rd64(dev->hw,
1772 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXPKTS]
1773 + stats_inst_offset_64);
1774 gather_stats->ip6txfrags =
1775 rd64(dev->hw,
1776 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXFRAGS]
1777 + stats_inst_offset_64);
1778 gather_stats->ip6rxmcpkts =
1779 rd64(dev->hw,
1780 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6RXMCPKTS]
1781 + stats_inst_offset_64);
1782 gather_stats->ip6txocts =
1783 rd64(dev->hw,
1784 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXOCTS]
1785 + stats_inst_offset_64);
1786 gather_stats->ip6txpkts =
1787 rd64(dev->hw,
1788 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXPKTS]
1789 + stats_inst_offset_64);
1790 gather_stats->ip6txfrags =
1791 rd64(dev->hw,
1792 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXFRAGS]
1793 + stats_inst_offset_64);
1794 gather_stats->ip6txmcpkts =
1795 rd64(dev->hw,
1796 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_IP6TXMCPKTS]
1797 + stats_inst_offset_64);
1798 gather_stats->tcprxsegs =
1799 rd64(dev->hw,
1800 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPRXSEGS]
1801 + stats_inst_offset_64);
1802 gather_stats->tcptxsegs =
1803 rd64(dev->hw,
1804 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_TCPTXSEG]
1805 + stats_inst_offset_64);
1806 gather_stats->rdmarxrds =
1807 rd64(dev->hw,
1808 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXRDS]
1809 + stats_inst_offset_64);
1810 gather_stats->rdmarxsnds =
1811 rd64(dev->hw,
1812 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXSNDS]
1813 + stats_inst_offset_64);
1814 gather_stats->rdmarxwrs =
1815 rd64(dev->hw,
1816 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMARXWRS]
1817 + stats_inst_offset_64);
1818 gather_stats->rdmatxrds =
1819 rd64(dev->hw,
1820 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXRDS]
1821 + stats_inst_offset_64);
1822 gather_stats->rdmatxsnds =
1823 rd64(dev->hw,
1824 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXSNDS]
1825 + stats_inst_offset_64);
1826 gather_stats->rdmatxwrs =
1827 rd64(dev->hw,
1828 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMATXWRS]
1829 + stats_inst_offset_64);
1830 gather_stats->rdmavbn =
1831 rd64(dev->hw,
1832 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVBND]
1833 + stats_inst_offset_64);
1834 gather_stats->rdmavinv =
1835 rd64(dev->hw,
1836 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_RDMAVINV]
1837 + stats_inst_offset_64);
1838 gather_stats->udprxpkts =
1839 rd64(dev->hw,
1840 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPRXPKTS]
1841 + stats_inst_offset_64);
1842 gather_stats->udptxpkts =
1843 rd64(dev->hw,
1844 dev->hw_stats_regs_64[IRDMA_HW_STAT_INDEX_UDPTXPKTS]
1845 + stats_inst_offset_64);
1846
1847 irdma_process_stats(pestat);
1848 }
1849
1850 /**
1851 * irdma_process_cqp_stats - Checking for wrap and update stats
1852 * @cqp_request: cqp_request structure pointer
1853 */
irdma_process_cqp_stats(struct irdma_cqp_request * cqp_request)1854 static void irdma_process_cqp_stats(struct irdma_cqp_request *cqp_request)
1855 {
1856 struct irdma_vsi_pestat *pestat = cqp_request->param;
1857
1858 irdma_process_stats(pestat);
1859 }
1860
1861 /**
1862 * irdma_cqp_gather_stats_cmd - Gather stats
1863 * @dev: pointer to device structure
1864 * @pestat: pointer to stats info
1865 * @wait: flag to wait or not wait for stats
1866 */
irdma_cqp_gather_stats_cmd(struct irdma_sc_dev * dev,struct irdma_vsi_pestat * pestat,bool wait)1867 int irdma_cqp_gather_stats_cmd(struct irdma_sc_dev *dev,
1868 struct irdma_vsi_pestat *pestat, bool wait)
1869
1870 {
1871 struct irdma_pci_f *rf = dev_to_rf(dev);
1872 struct irdma_cqp *iwcqp = &rf->cqp;
1873 struct irdma_cqp_request *cqp_request;
1874 struct cqp_cmds_info *cqp_info;
1875 int status;
1876
1877 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1878 if (!cqp_request)
1879 return -ENOMEM;
1880
1881 cqp_info = &cqp_request->info;
1882 memset(cqp_info, 0, sizeof(*cqp_info));
1883 cqp_info->cqp_cmd = IRDMA_OP_STATS_GATHER;
1884 cqp_info->post_sq = 1;
1885 cqp_info->in.u.stats_gather.info = pestat->gather_info;
1886 cqp_info->in.u.stats_gather.scratch = (uintptr_t)cqp_request;
1887 cqp_info->in.u.stats_gather.cqp = &rf->cqp.sc_cqp;
1888 cqp_request->param = pestat;
1889 if (!wait)
1890 cqp_request->callback_fcn = irdma_process_cqp_stats;
1891 status = irdma_handle_cqp_op(rf, cqp_request);
1892 if (wait)
1893 irdma_process_stats(pestat);
1894 irdma_put_cqp_request(&rf->cqp, cqp_request);
1895
1896 return status;
1897 }
1898
1899 /**
1900 * irdma_cqp_stats_inst_cmd - Allocate/free stats instance
1901 * @vsi: pointer to vsi structure
1902 * @cmd: command to allocate or free
1903 * @stats_info: pointer to allocate stats info
1904 */
irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi * vsi,u8 cmd,struct irdma_stats_inst_info * stats_info)1905 int irdma_cqp_stats_inst_cmd(struct irdma_sc_vsi *vsi, u8 cmd,
1906 struct irdma_stats_inst_info *stats_info)
1907 {
1908 struct irdma_pci_f *rf = dev_to_rf(vsi->dev);
1909 struct irdma_cqp *iwcqp = &rf->cqp;
1910 struct irdma_cqp_request *cqp_request;
1911 struct cqp_cmds_info *cqp_info;
1912 int status;
1913 bool wait = false;
1914
1915 if (cmd == IRDMA_OP_STATS_ALLOCATE)
1916 wait = true;
1917 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, wait);
1918 if (!cqp_request)
1919 return -ENOMEM;
1920
1921 cqp_info = &cqp_request->info;
1922 memset(cqp_info, 0, sizeof(*cqp_info));
1923 cqp_info->cqp_cmd = cmd;
1924 cqp_info->post_sq = 1;
1925 cqp_info->in.u.stats_manage.info = *stats_info;
1926 cqp_info->in.u.stats_manage.scratch = (uintptr_t)cqp_request;
1927 cqp_info->in.u.stats_manage.cqp = &rf->cqp.sc_cqp;
1928 status = irdma_handle_cqp_op(rf, cqp_request);
1929 if (wait)
1930 stats_info->stats_idx = cqp_request->compl_info.op_ret_val;
1931 irdma_put_cqp_request(iwcqp, cqp_request);
1932
1933 return status;
1934 }
1935
1936 /**
1937 * irdma_cqp_ceq_cmd - Create/Destroy CEQ's after CEQ 0
1938 * @dev: pointer to device info
1939 * @sc_ceq: pointer to ceq structure
1940 * @op: Create or Destroy
1941 */
irdma_cqp_ceq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_ceq * sc_ceq,u8 op)1942 int irdma_cqp_ceq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_ceq *sc_ceq,
1943 u8 op)
1944 {
1945 struct irdma_cqp_request *cqp_request;
1946 struct cqp_cmds_info *cqp_info;
1947 struct irdma_pci_f *rf = dev_to_rf(dev);
1948 int status;
1949
1950 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1951 if (!cqp_request)
1952 return -ENOMEM;
1953
1954 cqp_info = &cqp_request->info;
1955 cqp_info->post_sq = 1;
1956 cqp_info->cqp_cmd = op;
1957 cqp_info->in.u.ceq_create.ceq = sc_ceq;
1958 cqp_info->in.u.ceq_create.scratch = (uintptr_t)cqp_request;
1959
1960 status = irdma_handle_cqp_op(rf, cqp_request);
1961 irdma_put_cqp_request(&rf->cqp, cqp_request);
1962
1963 return status;
1964 }
1965
1966 /**
1967 * irdma_cqp_aeq_cmd - Create/Destroy AEQ
1968 * @dev: pointer to device info
1969 * @sc_aeq: pointer to aeq structure
1970 * @op: Create or Destroy
1971 */
irdma_cqp_aeq_cmd(struct irdma_sc_dev * dev,struct irdma_sc_aeq * sc_aeq,u8 op)1972 int irdma_cqp_aeq_cmd(struct irdma_sc_dev *dev, struct irdma_sc_aeq *sc_aeq,
1973 u8 op)
1974 {
1975 struct irdma_cqp_request *cqp_request;
1976 struct cqp_cmds_info *cqp_info;
1977 struct irdma_pci_f *rf = dev_to_rf(dev);
1978 int status;
1979
1980 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, true);
1981 if (!cqp_request)
1982 return -ENOMEM;
1983
1984 cqp_info = &cqp_request->info;
1985 cqp_info->post_sq = 1;
1986 cqp_info->cqp_cmd = op;
1987 cqp_info->in.u.aeq_create.aeq = sc_aeq;
1988 cqp_info->in.u.aeq_create.scratch = (uintptr_t)cqp_request;
1989
1990 status = irdma_handle_cqp_op(rf, cqp_request);
1991 irdma_put_cqp_request(&rf->cqp, cqp_request);
1992
1993 return status;
1994 }
1995
1996 /**
1997 * irdma_cqp_ws_node_cmd - Add/modify/delete ws node
1998 * @dev: pointer to device structure
1999 * @cmd: Add, modify or delete
2000 * @node_info: pointer to ws node info
2001 */
irdma_cqp_ws_node_cmd(struct irdma_sc_dev * dev,u8 cmd,struct irdma_ws_node_info * node_info)2002 int irdma_cqp_ws_node_cmd(struct irdma_sc_dev *dev, u8 cmd,
2003 struct irdma_ws_node_info *node_info)
2004 {
2005 struct irdma_pci_f *rf = dev_to_rf(dev);
2006 struct irdma_cqp *iwcqp = &rf->cqp;
2007 struct irdma_sc_cqp *cqp = &iwcqp->sc_cqp;
2008 struct irdma_cqp_request *cqp_request;
2009 struct cqp_cmds_info *cqp_info;
2010 int status;
2011 bool poll;
2012
2013 if (!rf->sc_dev.ceq_valid)
2014 poll = true;
2015 else
2016 poll = false;
2017
2018 cqp_request = irdma_alloc_and_get_cqp_request(iwcqp, !poll);
2019 if (!cqp_request)
2020 return -ENOMEM;
2021
2022 cqp_info = &cqp_request->info;
2023 memset(cqp_info, 0, sizeof(*cqp_info));
2024 cqp_info->cqp_cmd = cmd;
2025 cqp_info->post_sq = 1;
2026 cqp_info->in.u.ws_node.info = *node_info;
2027 cqp_info->in.u.ws_node.cqp = cqp;
2028 cqp_info->in.u.ws_node.scratch = (uintptr_t)cqp_request;
2029 status = irdma_handle_cqp_op(rf, cqp_request);
2030 if (status)
2031 goto exit;
2032
2033 if (poll) {
2034 struct irdma_ccq_cqe_info compl_info;
2035
2036 status = irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_WORK_SCHED_NODE,
2037 &compl_info);
2038 node_info->qs_handle = compl_info.op_ret_val;
2039 ibdev_dbg(&rf->iwdev->ibdev, "DCB: opcode=%d, compl_info.retval=%d\n",
2040 compl_info.op_code, compl_info.op_ret_val);
2041 } else {
2042 node_info->qs_handle = cqp_request->compl_info.op_ret_val;
2043 }
2044
2045 exit:
2046 irdma_put_cqp_request(&rf->cqp, cqp_request);
2047
2048 return status;
2049 }
2050
2051 /**
2052 * irdma_ah_cqp_op - perform an AH cqp operation
2053 * @rf: RDMA PCI function
2054 * @sc_ah: address handle
2055 * @cmd: AH operation
2056 * @wait: wait if true
2057 * @callback_fcn: Callback function on CQP op completion
2058 * @cb_param: parameter for callback function
2059 *
2060 * returns errno
2061 */
irdma_ah_cqp_op(struct irdma_pci_f * rf,struct irdma_sc_ah * sc_ah,u8 cmd,bool wait,void (* callback_fcn)(struct irdma_cqp_request *),void * cb_param)2062 int irdma_ah_cqp_op(struct irdma_pci_f *rf, struct irdma_sc_ah *sc_ah, u8 cmd,
2063 bool wait,
2064 void (*callback_fcn)(struct irdma_cqp_request *),
2065 void *cb_param)
2066 {
2067 struct irdma_cqp_request *cqp_request;
2068 struct cqp_cmds_info *cqp_info;
2069 int status;
2070
2071 if (cmd != IRDMA_OP_AH_CREATE && cmd != IRDMA_OP_AH_DESTROY)
2072 return -EINVAL;
2073
2074 cqp_request = irdma_alloc_and_get_cqp_request(&rf->cqp, wait);
2075 if (!cqp_request)
2076 return -ENOMEM;
2077
2078 cqp_info = &cqp_request->info;
2079 cqp_info->cqp_cmd = cmd;
2080 cqp_info->post_sq = 1;
2081 if (cmd == IRDMA_OP_AH_CREATE) {
2082 cqp_info->in.u.ah_create.info = sc_ah->ah_info;
2083 cqp_info->in.u.ah_create.scratch = (uintptr_t)cqp_request;
2084 cqp_info->in.u.ah_create.cqp = &rf->cqp.sc_cqp;
2085 } else if (cmd == IRDMA_OP_AH_DESTROY) {
2086 cqp_info->in.u.ah_destroy.info = sc_ah->ah_info;
2087 cqp_info->in.u.ah_destroy.scratch = (uintptr_t)cqp_request;
2088 cqp_info->in.u.ah_destroy.cqp = &rf->cqp.sc_cqp;
2089 }
2090
2091 if (!wait) {
2092 cqp_request->callback_fcn = callback_fcn;
2093 cqp_request->param = cb_param;
2094 }
2095 status = irdma_handle_cqp_op(rf, cqp_request);
2096 irdma_put_cqp_request(&rf->cqp, cqp_request);
2097
2098 if (status)
2099 return -ENOMEM;
2100
2101 if (wait)
2102 sc_ah->ah_info.ah_valid = (cmd == IRDMA_OP_AH_CREATE);
2103
2104 return 0;
2105 }
2106
2107 /**
2108 * irdma_ieq_ah_cb - callback after creation of AH for IEQ
2109 * @cqp_request: pointer to cqp_request of create AH
2110 */
irdma_ieq_ah_cb(struct irdma_cqp_request * cqp_request)2111 static void irdma_ieq_ah_cb(struct irdma_cqp_request *cqp_request)
2112 {
2113 struct irdma_sc_qp *qp = cqp_request->param;
2114 struct irdma_sc_ah *sc_ah = qp->pfpdu.ah;
2115 unsigned long flags;
2116
2117 spin_lock_irqsave(&qp->pfpdu.lock, flags);
2118 if (!cqp_request->compl_info.op_ret_val) {
2119 sc_ah->ah_info.ah_valid = true;
2120 irdma_ieq_process_fpdus(qp, qp->vsi->ieq);
2121 } else {
2122 sc_ah->ah_info.ah_valid = false;
2123 irdma_ieq_cleanup_qp(qp->vsi->ieq, qp);
2124 }
2125 spin_unlock_irqrestore(&qp->pfpdu.lock, flags);
2126 }
2127
2128 /**
2129 * irdma_ilq_ah_cb - callback after creation of AH for ILQ
2130 * @cqp_request: pointer to cqp_request of create AH
2131 */
irdma_ilq_ah_cb(struct irdma_cqp_request * cqp_request)2132 static void irdma_ilq_ah_cb(struct irdma_cqp_request *cqp_request)
2133 {
2134 struct irdma_cm_node *cm_node = cqp_request->param;
2135 struct irdma_sc_ah *sc_ah = cm_node->ah;
2136
2137 sc_ah->ah_info.ah_valid = !cqp_request->compl_info.op_ret_val;
2138 irdma_add_conn_est_qh(cm_node);
2139 }
2140
2141 /**
2142 * irdma_puda_create_ah - create AH for ILQ/IEQ qp's
2143 * @dev: device pointer
2144 * @ah_info: Address handle info
2145 * @wait: When true will wait for operation to complete
2146 * @type: ILQ/IEQ
2147 * @cb_param: Callback param when not waiting
2148 * @ah_ret: Returned pointer to address handle if created
2149 *
2150 */
irdma_puda_create_ah(struct irdma_sc_dev * dev,struct irdma_ah_info * ah_info,bool wait,enum puda_rsrc_type type,void * cb_param,struct irdma_sc_ah ** ah_ret)2151 int irdma_puda_create_ah(struct irdma_sc_dev *dev,
2152 struct irdma_ah_info *ah_info, bool wait,
2153 enum puda_rsrc_type type, void *cb_param,
2154 struct irdma_sc_ah **ah_ret)
2155 {
2156 struct irdma_sc_ah *ah;
2157 struct irdma_pci_f *rf = dev_to_rf(dev);
2158 int err;
2159
2160 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2161 *ah_ret = ah;
2162 if (!ah)
2163 return -ENOMEM;
2164
2165 err = irdma_alloc_rsrc(rf, rf->allocated_ahs, rf->max_ah,
2166 &ah_info->ah_idx, &rf->next_ah);
2167 if (err)
2168 goto err_free;
2169
2170 ah->dev = dev;
2171 ah->ah_info = *ah_info;
2172
2173 if (type == IRDMA_PUDA_RSRC_TYPE_ILQ)
2174 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2175 irdma_ilq_ah_cb, cb_param);
2176 else
2177 err = irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_CREATE, wait,
2178 irdma_ieq_ah_cb, cb_param);
2179
2180 if (err)
2181 goto error;
2182 return 0;
2183
2184 error:
2185 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2186 err_free:
2187 kfree(ah);
2188 *ah_ret = NULL;
2189 return -ENOMEM;
2190 }
2191
2192 /**
2193 * irdma_puda_free_ah - free a puda address handle
2194 * @dev: device pointer
2195 * @ah: The address handle to free
2196 */
irdma_puda_free_ah(struct irdma_sc_dev * dev,struct irdma_sc_ah * ah)2197 void irdma_puda_free_ah(struct irdma_sc_dev *dev, struct irdma_sc_ah *ah)
2198 {
2199 struct irdma_pci_f *rf = dev_to_rf(dev);
2200
2201 if (!ah)
2202 return;
2203
2204 if (ah->ah_info.ah_valid) {
2205 irdma_ah_cqp_op(rf, ah, IRDMA_OP_AH_DESTROY, false, NULL, NULL);
2206 irdma_free_rsrc(rf, rf->allocated_ahs, ah->ah_info.ah_idx);
2207 }
2208
2209 kfree(ah);
2210 }
2211
2212 /**
2213 * irdma_gsi_ud_qp_ah_cb - callback after creation of AH for GSI/ID QP
2214 * @cqp_request: pointer to cqp_request of create AH
2215 */
irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request * cqp_request)2216 void irdma_gsi_ud_qp_ah_cb(struct irdma_cqp_request *cqp_request)
2217 {
2218 struct irdma_sc_ah *sc_ah = cqp_request->param;
2219
2220 if (!cqp_request->compl_info.op_ret_val)
2221 sc_ah->ah_info.ah_valid = true;
2222 else
2223 sc_ah->ah_info.ah_valid = false;
2224 }
2225
2226 /**
2227 * irdma_prm_add_pble_mem - add moemory to pble resources
2228 * @pprm: pble resource manager
2229 * @pchunk: chunk of memory to add
2230 */
irdma_prm_add_pble_mem(struct irdma_pble_prm * pprm,struct irdma_chunk * pchunk)2231 int irdma_prm_add_pble_mem(struct irdma_pble_prm *pprm,
2232 struct irdma_chunk *pchunk)
2233 {
2234 u64 sizeofbitmap;
2235
2236 if (pchunk->size & 0xfff)
2237 return -EINVAL;
2238
2239 sizeofbitmap = (u64)pchunk->size >> pprm->pble_shift;
2240
2241 pchunk->bitmapbuf = bitmap_zalloc(sizeofbitmap, GFP_KERNEL);
2242 if (!pchunk->bitmapbuf)
2243 return -ENOMEM;
2244
2245 pchunk->sizeofbitmap = sizeofbitmap;
2246 /* each pble is 8 bytes hence shift by 3 */
2247 pprm->total_pble_alloc += pchunk->size >> 3;
2248 pprm->free_pble_cnt += pchunk->size >> 3;
2249
2250 return 0;
2251 }
2252
2253 /**
2254 * irdma_prm_get_pbles - get pble's from prm
2255 * @pprm: pble resource manager
2256 * @chunkinfo: nformation about chunk where pble's were acquired
2257 * @mem_size: size of pble memory needed
2258 * @vaddr: returns virtual address of pble memory
2259 * @fpm_addr: returns fpm address of pble memory
2260 */
irdma_prm_get_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo,u64 mem_size,u64 ** vaddr,u64 * fpm_addr)2261 int irdma_prm_get_pbles(struct irdma_pble_prm *pprm,
2262 struct irdma_pble_chunkinfo *chunkinfo, u64 mem_size,
2263 u64 **vaddr, u64 *fpm_addr)
2264 {
2265 u64 bits_needed;
2266 u64 bit_idx = PBLE_INVALID_IDX;
2267 struct irdma_chunk *pchunk = NULL;
2268 struct list_head *chunk_entry = pprm->clist.next;
2269 u32 offset;
2270 unsigned long flags;
2271 *vaddr = NULL;
2272 *fpm_addr = 0;
2273
2274 bits_needed = DIV_ROUND_UP_ULL(mem_size, BIT_ULL(pprm->pble_shift));
2275
2276 spin_lock_irqsave(&pprm->prm_lock, flags);
2277 while (chunk_entry != &pprm->clist) {
2278 pchunk = (struct irdma_chunk *)chunk_entry;
2279 bit_idx = bitmap_find_next_zero_area(pchunk->bitmapbuf,
2280 pchunk->sizeofbitmap, 0,
2281 bits_needed, 0);
2282 if (bit_idx < pchunk->sizeofbitmap)
2283 break;
2284
2285 /* list.next used macro */
2286 chunk_entry = pchunk->list.next;
2287 }
2288
2289 if (!pchunk || bit_idx >= pchunk->sizeofbitmap) {
2290 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2291 return -ENOMEM;
2292 }
2293
2294 bitmap_set(pchunk->bitmapbuf, bit_idx, bits_needed);
2295 offset = bit_idx << pprm->pble_shift;
2296 *vaddr = pchunk->vaddr + offset;
2297 *fpm_addr = pchunk->fpm_addr + offset;
2298
2299 chunkinfo->pchunk = pchunk;
2300 chunkinfo->bit_idx = bit_idx;
2301 chunkinfo->bits_used = bits_needed;
2302 /* 3 is sizeof pble divide */
2303 pprm->free_pble_cnt -= chunkinfo->bits_used << (pprm->pble_shift - 3);
2304 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2305
2306 return 0;
2307 }
2308
2309 /**
2310 * irdma_prm_return_pbles - return pbles back to prm
2311 * @pprm: pble resource manager
2312 * @chunkinfo: chunk where pble's were acquired and to be freed
2313 */
irdma_prm_return_pbles(struct irdma_pble_prm * pprm,struct irdma_pble_chunkinfo * chunkinfo)2314 void irdma_prm_return_pbles(struct irdma_pble_prm *pprm,
2315 struct irdma_pble_chunkinfo *chunkinfo)
2316 {
2317 unsigned long flags;
2318
2319 spin_lock_irqsave(&pprm->prm_lock, flags);
2320 pprm->free_pble_cnt += chunkinfo->bits_used << (pprm->pble_shift - 3);
2321 bitmap_clear(chunkinfo->pchunk->bitmapbuf, chunkinfo->bit_idx,
2322 chunkinfo->bits_used);
2323 spin_unlock_irqrestore(&pprm->prm_lock, flags);
2324 }
2325
irdma_map_vm_page_list(struct irdma_hw * hw,void * va,dma_addr_t * pg_dma,u32 pg_cnt)2326 int irdma_map_vm_page_list(struct irdma_hw *hw, void *va, dma_addr_t *pg_dma,
2327 u32 pg_cnt)
2328 {
2329 struct page *vm_page;
2330 int i;
2331 u8 *addr;
2332
2333 addr = (u8 *)(uintptr_t)va;
2334 for (i = 0; i < pg_cnt; i++) {
2335 vm_page = vmalloc_to_page(addr);
2336 if (!vm_page)
2337 goto err;
2338
2339 pg_dma[i] = dma_map_page(hw->device, vm_page, 0, PAGE_SIZE,
2340 DMA_BIDIRECTIONAL);
2341 if (dma_mapping_error(hw->device, pg_dma[i]))
2342 goto err;
2343
2344 addr += PAGE_SIZE;
2345 }
2346
2347 return 0;
2348
2349 err:
2350 irdma_unmap_vm_page_list(hw, pg_dma, i);
2351 return -ENOMEM;
2352 }
2353
irdma_unmap_vm_page_list(struct irdma_hw * hw,dma_addr_t * pg_dma,u32 pg_cnt)2354 void irdma_unmap_vm_page_list(struct irdma_hw *hw, dma_addr_t *pg_dma, u32 pg_cnt)
2355 {
2356 int i;
2357
2358 for (i = 0; i < pg_cnt; i++)
2359 dma_unmap_page(hw->device, pg_dma[i], PAGE_SIZE, DMA_BIDIRECTIONAL);
2360 }
2361
2362 /**
2363 * irdma_pble_free_paged_mem - free virtual paged memory
2364 * @chunk: chunk to free with paged memory
2365 */
irdma_pble_free_paged_mem(struct irdma_chunk * chunk)2366 void irdma_pble_free_paged_mem(struct irdma_chunk *chunk)
2367 {
2368 if (!chunk->pg_cnt)
2369 goto done;
2370
2371 irdma_unmap_vm_page_list(chunk->dev->hw, chunk->dmainfo.dmaaddrs,
2372 chunk->pg_cnt);
2373
2374 done:
2375 kfree(chunk->dmainfo.dmaaddrs);
2376 chunk->dmainfo.dmaaddrs = NULL;
2377 vfree(chunk->vaddr);
2378 chunk->vaddr = NULL;
2379 chunk->type = 0;
2380 }
2381
2382 /**
2383 * irdma_pble_get_paged_mem -allocate paged memory for pbles
2384 * @chunk: chunk to add for paged memory
2385 * @pg_cnt: number of pages needed
2386 */
irdma_pble_get_paged_mem(struct irdma_chunk * chunk,u32 pg_cnt)2387 int irdma_pble_get_paged_mem(struct irdma_chunk *chunk, u32 pg_cnt)
2388 {
2389 u32 size;
2390 void *va;
2391
2392 chunk->dmainfo.dmaaddrs = kzalloc(pg_cnt << 3, GFP_KERNEL);
2393 if (!chunk->dmainfo.dmaaddrs)
2394 return -ENOMEM;
2395
2396 size = PAGE_SIZE * pg_cnt;
2397 va = vmalloc(size);
2398 if (!va)
2399 goto err;
2400
2401 if (irdma_map_vm_page_list(chunk->dev->hw, va, chunk->dmainfo.dmaaddrs,
2402 pg_cnt)) {
2403 vfree(va);
2404 goto err;
2405 }
2406 chunk->vaddr = va;
2407 chunk->size = size;
2408 chunk->pg_cnt = pg_cnt;
2409 chunk->type = PBLE_SD_PAGED;
2410
2411 return 0;
2412 err:
2413 kfree(chunk->dmainfo.dmaaddrs);
2414 chunk->dmainfo.dmaaddrs = NULL;
2415
2416 return -ENOMEM;
2417 }
2418
2419 /**
2420 * irdma_alloc_ws_node_id - Allocate a tx scheduler node ID
2421 * @dev: device pointer
2422 */
irdma_alloc_ws_node_id(struct irdma_sc_dev * dev)2423 u16 irdma_alloc_ws_node_id(struct irdma_sc_dev *dev)
2424 {
2425 struct irdma_pci_f *rf = dev_to_rf(dev);
2426 u32 next = 1;
2427 u32 node_id;
2428
2429 if (irdma_alloc_rsrc(rf, rf->allocated_ws_nodes, rf->max_ws_node_id,
2430 &node_id, &next))
2431 return IRDMA_WS_NODE_INVALID;
2432
2433 return (u16)node_id;
2434 }
2435
2436 /**
2437 * irdma_free_ws_node_id - Free a tx scheduler node ID
2438 * @dev: device pointer
2439 * @node_id: Work scheduler node ID
2440 */
irdma_free_ws_node_id(struct irdma_sc_dev * dev,u16 node_id)2441 void irdma_free_ws_node_id(struct irdma_sc_dev *dev, u16 node_id)
2442 {
2443 struct irdma_pci_f *rf = dev_to_rf(dev);
2444
2445 irdma_free_rsrc(rf, rf->allocated_ws_nodes, (u32)node_id);
2446 }
2447
2448 /**
2449 * irdma_modify_qp_to_err - Modify a QP to error
2450 * @sc_qp: qp structure
2451 */
irdma_modify_qp_to_err(struct irdma_sc_qp * sc_qp)2452 void irdma_modify_qp_to_err(struct irdma_sc_qp *sc_qp)
2453 {
2454 struct irdma_qp *qp = sc_qp->qp_uk.back_qp;
2455 struct ib_qp_attr attr;
2456
2457 if (qp->iwdev->rf->reset)
2458 return;
2459 attr.qp_state = IB_QPS_ERR;
2460
2461 if (rdma_protocol_roce(qp->ibqp.device, 1))
2462 irdma_modify_qp_roce(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2463 else
2464 irdma_modify_qp(&qp->ibqp, &attr, IB_QP_STATE, NULL);
2465 }
2466
irdma_ib_qp_event(struct irdma_qp * iwqp,enum irdma_qp_event_type event)2467 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)
2468 {
2469 struct ib_event ibevent;
2470
2471 if (!iwqp->ibqp.event_handler)
2472 return;
2473
2474 switch (event) {
2475 case IRDMA_QP_EVENT_CATASTROPHIC:
2476 ibevent.event = IB_EVENT_QP_FATAL;
2477 break;
2478 case IRDMA_QP_EVENT_ACCESS_ERR:
2479 ibevent.event = IB_EVENT_QP_ACCESS_ERR;
2480 break;
2481 }
2482 ibevent.device = iwqp->ibqp.device;
2483 ibevent.element.qp = &iwqp->ibqp;
2484 iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);
2485 }
2486
irdma_cq_empty(struct irdma_cq * iwcq)2487 bool irdma_cq_empty(struct irdma_cq *iwcq)
2488 {
2489 struct irdma_cq_uk *ukcq;
2490 u64 qword3;
2491 __le64 *cqe;
2492 u8 polarity;
2493
2494 ukcq = &iwcq->sc_cq.cq_uk;
2495 cqe = IRDMA_GET_CURRENT_CQ_ELEM(ukcq);
2496 get_64bit_val(cqe, 24, &qword3);
2497 polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, qword3);
2498
2499 return polarity != ukcq->polarity;
2500 }
2501
irdma_remove_cmpls_list(struct irdma_cq * iwcq)2502 void irdma_remove_cmpls_list(struct irdma_cq *iwcq)
2503 {
2504 struct irdma_cmpl_gen *cmpl_node;
2505 struct list_head *tmp_node, *list_node;
2506
2507 list_for_each_safe (list_node, tmp_node, &iwcq->cmpl_generated) {
2508 cmpl_node = list_entry(list_node, struct irdma_cmpl_gen, list);
2509 list_del(&cmpl_node->list);
2510 kfree(cmpl_node);
2511 }
2512 }
2513
irdma_generated_cmpls(struct irdma_cq * iwcq,struct irdma_cq_poll_info * cq_poll_info)2514 int irdma_generated_cmpls(struct irdma_cq *iwcq, struct irdma_cq_poll_info *cq_poll_info)
2515 {
2516 struct irdma_cmpl_gen *cmpl;
2517
2518 if (list_empty(&iwcq->cmpl_generated))
2519 return -ENOENT;
2520 cmpl = list_first_entry_or_null(&iwcq->cmpl_generated, struct irdma_cmpl_gen, list);
2521 list_del(&cmpl->list);
2522 memcpy(cq_poll_info, &cmpl->cpi, sizeof(*cq_poll_info));
2523 kfree(cmpl);
2524
2525 ibdev_dbg(iwcq->ibcq.device,
2526 "VERBS: %s: Poll artificially generated completion for QP 0x%X, op %u, wr_id=0x%llx\n",
2527 __func__, cq_poll_info->qp_id, cq_poll_info->op_type,
2528 cq_poll_info->wr_id);
2529
2530 return 0;
2531 }
2532
2533 /**
2534 * irdma_set_cpi_common_values - fill in values for polling info struct
2535 * @cpi: resulting structure of cq_poll_info type
2536 * @qp: QPair
2537 * @qp_num: id of the QP
2538 */
irdma_set_cpi_common_values(struct irdma_cq_poll_info * cpi,struct irdma_qp_uk * qp,u32 qp_num)2539 static void irdma_set_cpi_common_values(struct irdma_cq_poll_info *cpi,
2540 struct irdma_qp_uk *qp, u32 qp_num)
2541 {
2542 cpi->comp_status = IRDMA_COMPL_STATUS_FLUSHED;
2543 cpi->error = true;
2544 cpi->major_err = IRDMA_FLUSH_MAJOR_ERR;
2545 cpi->minor_err = FLUSH_GENERAL_ERR;
2546 cpi->qp_handle = (irdma_qp_handle)(uintptr_t)qp;
2547 cpi->qp_id = qp_num;
2548 }
2549
irdma_comp_handler(struct irdma_cq * cq)2550 static inline void irdma_comp_handler(struct irdma_cq *cq)
2551 {
2552 if (!cq->ibcq.comp_handler)
2553 return;
2554 if (atomic_cmpxchg(&cq->armed, 1, 0))
2555 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
2556 }
2557
irdma_generate_flush_completions(struct irdma_qp * iwqp)2558 void irdma_generate_flush_completions(struct irdma_qp *iwqp)
2559 {
2560 struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;
2561 struct irdma_ring *sq_ring = &qp->sq_ring;
2562 struct irdma_ring *rq_ring = &qp->rq_ring;
2563 struct irdma_cmpl_gen *cmpl;
2564 __le64 *sw_wqe;
2565 u64 wqe_qword;
2566 u32 wqe_idx;
2567 bool compl_generated = false;
2568 unsigned long flags1;
2569
2570 spin_lock_irqsave(&iwqp->iwscq->lock, flags1);
2571 if (irdma_cq_empty(iwqp->iwscq)) {
2572 unsigned long flags2;
2573
2574 spin_lock_irqsave(&iwqp->lock, flags2);
2575 while (IRDMA_RING_MORE_WORK(*sq_ring)) {
2576 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2577 if (!cmpl) {
2578 spin_unlock_irqrestore(&iwqp->lock, flags2);
2579 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2580 return;
2581 }
2582
2583 wqe_idx = sq_ring->tail;
2584 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2585
2586 cmpl->cpi.wr_id = qp->sq_wrtrk_array[wqe_idx].wrid;
2587 sw_wqe = qp->sq_base[wqe_idx].elem;
2588 get_64bit_val(sw_wqe, 24, &wqe_qword);
2589 cmpl->cpi.op_type = (u8)FIELD_GET(IRDMAQPSQ_OPCODE, IRDMAQPSQ_OPCODE);
2590 /* remove the SQ WR by moving SQ tail*/
2591 IRDMA_RING_SET_TAIL(*sq_ring,
2592 sq_ring->tail + qp->sq_wrtrk_array[sq_ring->tail].quanta);
2593
2594 ibdev_dbg(iwqp->iwscq->ibcq.device,
2595 "DEV: %s: adding wr_id = 0x%llx SQ Completion to list qp_id=%d\n",
2596 __func__, cmpl->cpi.wr_id, qp->qp_id);
2597 list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);
2598 compl_generated = true;
2599 }
2600 spin_unlock_irqrestore(&iwqp->lock, flags2);
2601 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2602 if (compl_generated)
2603 irdma_comp_handler(iwqp->iwscq);
2604 } else {
2605 spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);
2606 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2607 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2608 }
2609
2610 spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);
2611 if (irdma_cq_empty(iwqp->iwrcq)) {
2612 unsigned long flags2;
2613
2614 spin_lock_irqsave(&iwqp->lock, flags2);
2615 while (IRDMA_RING_MORE_WORK(*rq_ring)) {
2616 cmpl = kzalloc(sizeof(*cmpl), GFP_ATOMIC);
2617 if (!cmpl) {
2618 spin_unlock_irqrestore(&iwqp->lock, flags2);
2619 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2620 return;
2621 }
2622
2623 wqe_idx = rq_ring->tail;
2624 irdma_set_cpi_common_values(&cmpl->cpi, qp, qp->qp_id);
2625
2626 cmpl->cpi.wr_id = qp->rq_wrid_array[wqe_idx];
2627 cmpl->cpi.op_type = IRDMA_OP_TYPE_REC;
2628 /* remove the RQ WR by moving RQ tail */
2629 IRDMA_RING_SET_TAIL(*rq_ring, rq_ring->tail + 1);
2630 ibdev_dbg(iwqp->iwrcq->ibcq.device,
2631 "DEV: %s: adding wr_id = 0x%llx RQ Completion to list qp_id=%d, wqe_idx=%d\n",
2632 __func__, cmpl->cpi.wr_id, qp->qp_id,
2633 wqe_idx);
2634 list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);
2635
2636 compl_generated = true;
2637 }
2638 spin_unlock_irqrestore(&iwqp->lock, flags2);
2639 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2640 if (compl_generated)
2641 irdma_comp_handler(iwqp->iwrcq);
2642 } else {
2643 spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);
2644 mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,
2645 msecs_to_jiffies(IRDMA_FLUSH_DELAY_MS));
2646 }
2647 }
2648