1 // SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
2 /*
3 * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
4 */
5
6 #include <linux/module.h>
7 #include <linux/pci.h>
8 #include <linux/utsname.h>
9 #include <linux/version.h>
10
11 #include <rdma/ib_user_verbs.h>
12
13 #include "efa.h"
14
15 #define PCI_DEV_ID_EFA0_VF 0xefa0
16 #define PCI_DEV_ID_EFA1_VF 0xefa1
17
18 static const struct pci_device_id efa_pci_tbl[] = {
19 { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA0_VF) },
20 { PCI_VDEVICE(AMAZON, PCI_DEV_ID_EFA1_VF) },
21 { }
22 };
23
24 MODULE_AUTHOR("Amazon.com, Inc. or its affiliates");
25 MODULE_LICENSE("Dual BSD/GPL");
26 MODULE_DESCRIPTION(DEVICE_NAME);
27 MODULE_DEVICE_TABLE(pci, efa_pci_tbl);
28
29 #define EFA_REG_BAR 0
30 #define EFA_MEM_BAR 2
31 #define EFA_BASE_BAR_MASK (BIT(EFA_REG_BAR) | BIT(EFA_MEM_BAR))
32
33 #define EFA_AENQ_ENABLED_GROUPS \
34 (BIT(EFA_ADMIN_FATAL_ERROR) | BIT(EFA_ADMIN_WARNING) | \
35 BIT(EFA_ADMIN_NOTIFICATION) | BIT(EFA_ADMIN_KEEP_ALIVE))
36
37 /* This handler will called for unknown event group or unimplemented handlers */
unimplemented_aenq_handler(void * data,struct efa_admin_aenq_entry * aenq_e)38 static void unimplemented_aenq_handler(void *data,
39 struct efa_admin_aenq_entry *aenq_e)
40 {
41 struct efa_dev *dev = (struct efa_dev *)data;
42
43 ibdev_err(&dev->ibdev,
44 "Unknown event was received or event with unimplemented handler\n");
45 }
46
efa_keep_alive(void * data,struct efa_admin_aenq_entry * aenq_e)47 static void efa_keep_alive(void *data, struct efa_admin_aenq_entry *aenq_e)
48 {
49 struct efa_dev *dev = (struct efa_dev *)data;
50
51 atomic64_inc(&dev->stats.keep_alive_rcvd);
52 }
53
54 static struct efa_aenq_handlers aenq_handlers = {
55 .handlers = {
56 [EFA_ADMIN_KEEP_ALIVE] = efa_keep_alive,
57 },
58 .unimplemented_handler = unimplemented_aenq_handler
59 };
60
efa_release_bars(struct efa_dev * dev,int bars_mask)61 static void efa_release_bars(struct efa_dev *dev, int bars_mask)
62 {
63 struct pci_dev *pdev = dev->pdev;
64 int release_bars;
65
66 release_bars = pci_select_bars(pdev, IORESOURCE_MEM) & bars_mask;
67 pci_release_selected_regions(pdev, release_bars);
68 }
69
efa_process_comp_eqe(struct efa_dev * dev,struct efa_admin_eqe * eqe)70 static void efa_process_comp_eqe(struct efa_dev *dev, struct efa_admin_eqe *eqe)
71 {
72 u16 cqn = eqe->u.comp_event.cqn;
73 struct efa_cq *cq;
74
75 /* Safe to load as we're in irq and removal calls synchronize_irq() */
76 cq = xa_load(&dev->cqs_xa, cqn);
77 if (unlikely(!cq)) {
78 ibdev_err_ratelimited(&dev->ibdev,
79 "Completion event on non-existent CQ[%u]",
80 cqn);
81 return;
82 }
83
84 cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context);
85 }
86
efa_process_eqe(struct efa_com_eq * eeq,struct efa_admin_eqe * eqe)87 static void efa_process_eqe(struct efa_com_eq *eeq, struct efa_admin_eqe *eqe)
88 {
89 struct efa_dev *dev = container_of(eeq->edev, struct efa_dev, edev);
90
91 if (likely(EFA_GET(&eqe->common, EFA_ADMIN_EQE_EVENT_TYPE) ==
92 EFA_ADMIN_EQE_EVENT_TYPE_COMPLETION))
93 efa_process_comp_eqe(dev, eqe);
94 else
95 ibdev_err_ratelimited(&dev->ibdev,
96 "Unknown event type received %lu",
97 EFA_GET(&eqe->common,
98 EFA_ADMIN_EQE_EVENT_TYPE));
99 }
100
efa_intr_msix_comp(int irq,void * data)101 static irqreturn_t efa_intr_msix_comp(int irq, void *data)
102 {
103 struct efa_eq *eq = data;
104 struct efa_com_dev *edev = eq->eeq.edev;
105
106 efa_com_eq_comp_intr_handler(edev, &eq->eeq);
107
108 return IRQ_HANDLED;
109 }
110
efa_intr_msix_mgmnt(int irq,void * data)111 static irqreturn_t efa_intr_msix_mgmnt(int irq, void *data)
112 {
113 struct efa_dev *dev = data;
114
115 efa_com_admin_q_comp_intr_handler(&dev->edev);
116 efa_com_aenq_intr_handler(&dev->edev, data);
117
118 return IRQ_HANDLED;
119 }
120
efa_request_irq(struct efa_dev * dev,struct efa_irq * irq)121 static int efa_request_irq(struct efa_dev *dev, struct efa_irq *irq)
122 {
123 int err;
124
125 err = request_irq(irq->irqn, irq->handler, 0, irq->name, irq->data);
126 if (err) {
127 dev_err(&dev->pdev->dev, "Failed to request irq %s (%d)\n",
128 irq->name, err);
129 return err;
130 }
131
132 irq_set_affinity_hint(irq->irqn, &irq->affinity_hint_mask);
133
134 return 0;
135 }
136
efa_setup_comp_irq(struct efa_dev * dev,struct efa_eq * eq,int vector)137 static void efa_setup_comp_irq(struct efa_dev *dev, struct efa_eq *eq,
138 int vector)
139 {
140 u32 cpu;
141
142 cpu = vector - EFA_COMP_EQS_VEC_BASE;
143 snprintf(eq->irq.name, EFA_IRQNAME_SIZE, "efa-comp%d@pci:%s", cpu,
144 pci_name(dev->pdev));
145 eq->irq.handler = efa_intr_msix_comp;
146 eq->irq.data = eq;
147 eq->irq.vector = vector;
148 eq->irq.irqn = pci_irq_vector(dev->pdev, vector);
149 cpumask_set_cpu(cpu, &eq->irq.affinity_hint_mask);
150 }
151
efa_free_irq(struct efa_dev * dev,struct efa_irq * irq)152 static void efa_free_irq(struct efa_dev *dev, struct efa_irq *irq)
153 {
154 irq_set_affinity_hint(irq->irqn, NULL);
155 free_irq(irq->irqn, irq->data);
156 }
157
efa_setup_mgmnt_irq(struct efa_dev * dev)158 static void efa_setup_mgmnt_irq(struct efa_dev *dev)
159 {
160 u32 cpu;
161
162 snprintf(dev->admin_irq.name, EFA_IRQNAME_SIZE,
163 "efa-mgmnt@pci:%s", pci_name(dev->pdev));
164 dev->admin_irq.handler = efa_intr_msix_mgmnt;
165 dev->admin_irq.data = dev;
166 dev->admin_irq.vector = dev->admin_msix_vector_idx;
167 dev->admin_irq.irqn = pci_irq_vector(dev->pdev,
168 dev->admin_msix_vector_idx);
169 cpu = cpumask_first(cpu_online_mask);
170 cpumask_set_cpu(cpu,
171 &dev->admin_irq.affinity_hint_mask);
172 dev_info(&dev->pdev->dev, "Setup irq:%d name:%s\n",
173 dev->admin_irq.irqn,
174 dev->admin_irq.name);
175 }
176
efa_set_mgmnt_irq(struct efa_dev * dev)177 static int efa_set_mgmnt_irq(struct efa_dev *dev)
178 {
179 efa_setup_mgmnt_irq(dev);
180
181 return efa_request_irq(dev, &dev->admin_irq);
182 }
183
efa_request_doorbell_bar(struct efa_dev * dev)184 static int efa_request_doorbell_bar(struct efa_dev *dev)
185 {
186 u8 db_bar_idx = dev->dev_attr.db_bar;
187 struct pci_dev *pdev = dev->pdev;
188 int bars;
189 int err;
190
191 if (!(BIT(db_bar_idx) & EFA_BASE_BAR_MASK)) {
192 bars = pci_select_bars(pdev, IORESOURCE_MEM) & BIT(db_bar_idx);
193
194 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
195 if (err) {
196 dev_err(&dev->pdev->dev,
197 "pci_request_selected_regions for bar %d failed %d\n",
198 db_bar_idx, err);
199 return err;
200 }
201 }
202
203 dev->db_bar_addr = pci_resource_start(dev->pdev, db_bar_idx);
204 dev->db_bar_len = pci_resource_len(dev->pdev, db_bar_idx);
205
206 return 0;
207 }
208
efa_release_doorbell_bar(struct efa_dev * dev)209 static void efa_release_doorbell_bar(struct efa_dev *dev)
210 {
211 if (!(BIT(dev->dev_attr.db_bar) & EFA_BASE_BAR_MASK))
212 efa_release_bars(dev, BIT(dev->dev_attr.db_bar));
213 }
214
efa_update_hw_hints(struct efa_dev * dev,struct efa_com_get_hw_hints_result * hw_hints)215 static void efa_update_hw_hints(struct efa_dev *dev,
216 struct efa_com_get_hw_hints_result *hw_hints)
217 {
218 struct efa_com_dev *edev = &dev->edev;
219
220 if (hw_hints->mmio_read_timeout)
221 edev->mmio_read.mmio_read_timeout =
222 hw_hints->mmio_read_timeout * 1000;
223
224 if (hw_hints->poll_interval)
225 edev->aq.poll_interval = hw_hints->poll_interval;
226
227 if (hw_hints->admin_completion_timeout)
228 edev->aq.completion_timeout =
229 hw_hints->admin_completion_timeout;
230 }
231
efa_stats_init(struct efa_dev * dev)232 static void efa_stats_init(struct efa_dev *dev)
233 {
234 atomic64_t *s = (atomic64_t *)&dev->stats;
235 int i;
236
237 for (i = 0; i < sizeof(dev->stats) / sizeof(*s); i++, s++)
238 atomic64_set(s, 0);
239 }
240
efa_set_host_info(struct efa_dev * dev)241 static void efa_set_host_info(struct efa_dev *dev)
242 {
243 struct efa_admin_set_feature_resp resp = {};
244 struct efa_admin_set_feature_cmd cmd = {};
245 struct efa_admin_host_info *hinf;
246 u32 bufsz = sizeof(*hinf);
247 dma_addr_t hinf_dma;
248
249 if (!efa_com_check_supported_feature_id(&dev->edev,
250 EFA_ADMIN_HOST_INFO))
251 return;
252
253 /* Failures in host info set shall not disturb probe */
254 hinf = dma_alloc_coherent(&dev->pdev->dev, bufsz, &hinf_dma,
255 GFP_KERNEL);
256 if (!hinf)
257 return;
258
259 strscpy(hinf->os_dist_str, utsname()->release,
260 sizeof(hinf->os_dist_str));
261 hinf->os_type = EFA_ADMIN_OS_LINUX;
262 strscpy(hinf->kernel_ver_str, utsname()->version,
263 sizeof(hinf->kernel_ver_str));
264 hinf->kernel_ver = LINUX_VERSION_CODE;
265 EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MAJOR, 0);
266 EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MINOR, 0);
267 EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_SUB_MINOR, 0);
268 EFA_SET(&hinf->driver_ver, EFA_ADMIN_HOST_INFO_DRIVER_MODULE_TYPE, 0);
269 EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_BUS, dev->pdev->bus->number);
270 EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_DEVICE,
271 PCI_SLOT(dev->pdev->devfn));
272 EFA_SET(&hinf->bdf, EFA_ADMIN_HOST_INFO_FUNCTION,
273 PCI_FUNC(dev->pdev->devfn));
274 EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MAJOR,
275 EFA_COMMON_SPEC_VERSION_MAJOR);
276 EFA_SET(&hinf->spec_ver, EFA_ADMIN_HOST_INFO_SPEC_MINOR,
277 EFA_COMMON_SPEC_VERSION_MINOR);
278 EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_INTREE, 1);
279 EFA_SET(&hinf->flags, EFA_ADMIN_HOST_INFO_GDR, 0);
280
281 efa_com_set_feature_ex(&dev->edev, &resp, &cmd, EFA_ADMIN_HOST_INFO,
282 hinf_dma, bufsz);
283
284 dma_free_coherent(&dev->pdev->dev, bufsz, hinf, hinf_dma);
285 }
286
efa_destroy_eq(struct efa_dev * dev,struct efa_eq * eq)287 static void efa_destroy_eq(struct efa_dev *dev, struct efa_eq *eq)
288 {
289 efa_com_eq_destroy(&dev->edev, &eq->eeq);
290 efa_free_irq(dev, &eq->irq);
291 }
292
efa_create_eq(struct efa_dev * dev,struct efa_eq * eq,u8 msix_vec)293 static int efa_create_eq(struct efa_dev *dev, struct efa_eq *eq, u8 msix_vec)
294 {
295 int err;
296
297 efa_setup_comp_irq(dev, eq, msix_vec);
298 err = efa_request_irq(dev, &eq->irq);
299 if (err)
300 return err;
301
302 err = efa_com_eq_init(&dev->edev, &eq->eeq, efa_process_eqe,
303 dev->dev_attr.max_eq_depth, msix_vec);
304 if (err)
305 goto err_free_comp_irq;
306
307 return 0;
308
309 err_free_comp_irq:
310 efa_free_irq(dev, &eq->irq);
311 return err;
312 }
313
efa_create_eqs(struct efa_dev * dev)314 static int efa_create_eqs(struct efa_dev *dev)
315 {
316 unsigned int neqs = dev->dev_attr.max_eq;
317 int err;
318 int i;
319
320 neqs = min_t(unsigned int, neqs, num_online_cpus());
321 dev->neqs = neqs;
322 dev->eqs = kcalloc(neqs, sizeof(*dev->eqs), GFP_KERNEL);
323 if (!dev->eqs)
324 return -ENOMEM;
325
326 for (i = 0; i < neqs; i++) {
327 err = efa_create_eq(dev, &dev->eqs[i],
328 i + EFA_COMP_EQS_VEC_BASE);
329 if (err)
330 goto err_destroy_eqs;
331 }
332
333 return 0;
334
335 err_destroy_eqs:
336 for (i--; i >= 0; i--)
337 efa_destroy_eq(dev, &dev->eqs[i]);
338 kfree(dev->eqs);
339
340 return err;
341 }
342
efa_destroy_eqs(struct efa_dev * dev)343 static void efa_destroy_eqs(struct efa_dev *dev)
344 {
345 int i;
346
347 for (i = 0; i < dev->neqs; i++)
348 efa_destroy_eq(dev, &dev->eqs[i]);
349
350 kfree(dev->eqs);
351 }
352
353 static const struct ib_device_ops efa_dev_ops = {
354 .owner = THIS_MODULE,
355 .driver_id = RDMA_DRIVER_EFA,
356 .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION,
357
358 .alloc_hw_port_stats = efa_alloc_hw_port_stats,
359 .alloc_hw_device_stats = efa_alloc_hw_device_stats,
360 .alloc_pd = efa_alloc_pd,
361 .alloc_ucontext = efa_alloc_ucontext,
362 .create_cq = efa_create_cq,
363 .create_qp = efa_create_qp,
364 .create_user_ah = efa_create_ah,
365 .dealloc_pd = efa_dealloc_pd,
366 .dealloc_ucontext = efa_dealloc_ucontext,
367 .dereg_mr = efa_dereg_mr,
368 .destroy_ah = efa_destroy_ah,
369 .destroy_cq = efa_destroy_cq,
370 .destroy_qp = efa_destroy_qp,
371 .get_hw_stats = efa_get_hw_stats,
372 .get_link_layer = efa_port_link_layer,
373 .get_port_immutable = efa_get_port_immutable,
374 .mmap = efa_mmap,
375 .mmap_free = efa_mmap_free,
376 .modify_qp = efa_modify_qp,
377 .query_device = efa_query_device,
378 .query_gid = efa_query_gid,
379 .query_pkey = efa_query_pkey,
380 .query_port = efa_query_port,
381 .query_qp = efa_query_qp,
382 .reg_user_mr = efa_reg_mr,
383 .reg_user_mr_dmabuf = efa_reg_user_mr_dmabuf,
384
385 INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah),
386 INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq),
387 INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd),
388 INIT_RDMA_OBJ_SIZE(ib_qp, efa_qp, ibqp),
389 INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext),
390 };
391
efa_ib_device_add(struct efa_dev * dev)392 static int efa_ib_device_add(struct efa_dev *dev)
393 {
394 struct efa_com_get_hw_hints_result hw_hints;
395 struct pci_dev *pdev = dev->pdev;
396 int err;
397
398 efa_stats_init(dev);
399
400 err = efa_com_get_device_attr(&dev->edev, &dev->dev_attr);
401 if (err)
402 return err;
403
404 dev_dbg(&dev->pdev->dev, "Doorbells bar (%d)\n", dev->dev_attr.db_bar);
405 err = efa_request_doorbell_bar(dev);
406 if (err)
407 return err;
408
409 err = efa_com_get_hw_hints(&dev->edev, &hw_hints);
410 if (err)
411 goto err_release_doorbell_bar;
412
413 efa_update_hw_hints(dev, &hw_hints);
414
415 /* Try to enable all the available aenq groups */
416 err = efa_com_set_aenq_config(&dev->edev, EFA_AENQ_ENABLED_GROUPS);
417 if (err)
418 goto err_release_doorbell_bar;
419
420 err = efa_create_eqs(dev);
421 if (err)
422 goto err_release_doorbell_bar;
423
424 efa_set_host_info(dev);
425
426 dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED;
427 dev->ibdev.phys_port_cnt = 1;
428 dev->ibdev.num_comp_vectors = dev->neqs ?: 1;
429 dev->ibdev.dev.parent = &pdev->dev;
430
431 ib_set_device_ops(&dev->ibdev, &efa_dev_ops);
432
433 err = ib_register_device(&dev->ibdev, "efa_%d", &pdev->dev);
434 if (err)
435 goto err_destroy_eqs;
436
437 ibdev_info(&dev->ibdev, "IB device registered\n");
438
439 return 0;
440
441 err_destroy_eqs:
442 efa_destroy_eqs(dev);
443 err_release_doorbell_bar:
444 efa_release_doorbell_bar(dev);
445 return err;
446 }
447
efa_ib_device_remove(struct efa_dev * dev)448 static void efa_ib_device_remove(struct efa_dev *dev)
449 {
450 ibdev_info(&dev->ibdev, "Unregister ib device\n");
451 ib_unregister_device(&dev->ibdev);
452 efa_destroy_eqs(dev);
453 efa_com_dev_reset(&dev->edev, EFA_REGS_RESET_NORMAL);
454 efa_release_doorbell_bar(dev);
455 }
456
efa_disable_msix(struct efa_dev * dev)457 static void efa_disable_msix(struct efa_dev *dev)
458 {
459 pci_free_irq_vectors(dev->pdev);
460 }
461
efa_enable_msix(struct efa_dev * dev)462 static int efa_enable_msix(struct efa_dev *dev)
463 {
464 int msix_vecs, irq_num;
465
466 /*
467 * Reserve the max msix vectors we might need, one vector is reserved
468 * for admin.
469 */
470 msix_vecs = min_t(int, pci_msix_vec_count(dev->pdev),
471 num_online_cpus() + 1);
472 dev_dbg(&dev->pdev->dev, "Trying to enable MSI-X, vectors %d\n",
473 msix_vecs);
474
475 dev->admin_msix_vector_idx = EFA_MGMNT_MSIX_VEC_IDX;
476 irq_num = pci_alloc_irq_vectors(dev->pdev, msix_vecs,
477 msix_vecs, PCI_IRQ_MSIX);
478
479 if (irq_num < 0) {
480 dev_err(&dev->pdev->dev, "Failed to enable MSI-X. irq_num %d\n",
481 irq_num);
482 return -ENOSPC;
483 }
484
485 if (irq_num != msix_vecs) {
486 efa_disable_msix(dev);
487 dev_err(&dev->pdev->dev,
488 "Allocated %d MSI-X (out of %d requested)\n",
489 irq_num, msix_vecs);
490 return -ENOSPC;
491 }
492
493 return 0;
494 }
495
efa_device_init(struct efa_com_dev * edev,struct pci_dev * pdev)496 static int efa_device_init(struct efa_com_dev *edev, struct pci_dev *pdev)
497 {
498 int dma_width;
499 int err;
500
501 err = efa_com_dev_reset(edev, EFA_REGS_RESET_NORMAL);
502 if (err)
503 return err;
504
505 err = efa_com_validate_version(edev);
506 if (err)
507 return err;
508
509 dma_width = efa_com_get_dma_width(edev);
510 if (dma_width < 0) {
511 err = dma_width;
512 return err;
513 }
514
515 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(dma_width));
516 if (err) {
517 dev_err(&pdev->dev, "dma_set_mask_and_coherent failed %d\n", err);
518 return err;
519 }
520
521 dma_set_max_seg_size(&pdev->dev, UINT_MAX);
522 return 0;
523 }
524
efa_probe_device(struct pci_dev * pdev)525 static struct efa_dev *efa_probe_device(struct pci_dev *pdev)
526 {
527 struct efa_com_dev *edev;
528 struct efa_dev *dev;
529 int bars;
530 int err;
531
532 err = pci_enable_device_mem(pdev);
533 if (err) {
534 dev_err(&pdev->dev, "pci_enable_device_mem() failed!\n");
535 return ERR_PTR(err);
536 }
537
538 pci_set_master(pdev);
539
540 dev = ib_alloc_device(efa_dev, ibdev);
541 if (!dev) {
542 dev_err(&pdev->dev, "Device alloc failed\n");
543 err = -ENOMEM;
544 goto err_disable_device;
545 }
546
547 pci_set_drvdata(pdev, dev);
548 edev = &dev->edev;
549 edev->efa_dev = dev;
550 edev->dmadev = &pdev->dev;
551 dev->pdev = pdev;
552 xa_init(&dev->cqs_xa);
553
554 bars = pci_select_bars(pdev, IORESOURCE_MEM) & EFA_BASE_BAR_MASK;
555 err = pci_request_selected_regions(pdev, bars, DRV_MODULE_NAME);
556 if (err) {
557 dev_err(&pdev->dev, "pci_request_selected_regions failed %d\n",
558 err);
559 goto err_ibdev_destroy;
560 }
561
562 dev->reg_bar_addr = pci_resource_start(pdev, EFA_REG_BAR);
563 dev->reg_bar_len = pci_resource_len(pdev, EFA_REG_BAR);
564 dev->mem_bar_addr = pci_resource_start(pdev, EFA_MEM_BAR);
565 dev->mem_bar_len = pci_resource_len(pdev, EFA_MEM_BAR);
566
567 edev->reg_bar = devm_ioremap(&pdev->dev,
568 dev->reg_bar_addr,
569 dev->reg_bar_len);
570 if (!edev->reg_bar) {
571 dev_err(&pdev->dev, "Failed to remap register bar\n");
572 err = -EFAULT;
573 goto err_release_bars;
574 }
575
576 err = efa_com_mmio_reg_read_init(edev);
577 if (err) {
578 dev_err(&pdev->dev, "Failed to init readless MMIO\n");
579 goto err_iounmap;
580 }
581
582 err = efa_device_init(edev, pdev);
583 if (err) {
584 dev_err(&pdev->dev, "EFA device init failed\n");
585 if (err == -ETIME)
586 err = -EPROBE_DEFER;
587 goto err_reg_read_destroy;
588 }
589
590 err = efa_enable_msix(dev);
591 if (err)
592 goto err_reg_read_destroy;
593
594 edev->aq.msix_vector_idx = dev->admin_msix_vector_idx;
595 edev->aenq.msix_vector_idx = dev->admin_msix_vector_idx;
596
597 err = efa_set_mgmnt_irq(dev);
598 if (err)
599 goto err_disable_msix;
600
601 err = efa_com_admin_init(edev, &aenq_handlers);
602 if (err)
603 goto err_free_mgmnt_irq;
604
605 return dev;
606
607 err_free_mgmnt_irq:
608 efa_free_irq(dev, &dev->admin_irq);
609 err_disable_msix:
610 efa_disable_msix(dev);
611 err_reg_read_destroy:
612 efa_com_mmio_reg_read_destroy(edev);
613 err_iounmap:
614 devm_iounmap(&pdev->dev, edev->reg_bar);
615 err_release_bars:
616 efa_release_bars(dev, EFA_BASE_BAR_MASK);
617 err_ibdev_destroy:
618 ib_dealloc_device(&dev->ibdev);
619 err_disable_device:
620 pci_disable_device(pdev);
621 return ERR_PTR(err);
622 }
623
efa_remove_device(struct pci_dev * pdev)624 static void efa_remove_device(struct pci_dev *pdev)
625 {
626 struct efa_dev *dev = pci_get_drvdata(pdev);
627 struct efa_com_dev *edev;
628
629 edev = &dev->edev;
630 efa_com_admin_destroy(edev);
631 efa_free_irq(dev, &dev->admin_irq);
632 efa_disable_msix(dev);
633 efa_com_mmio_reg_read_destroy(edev);
634 devm_iounmap(&pdev->dev, edev->reg_bar);
635 efa_release_bars(dev, EFA_BASE_BAR_MASK);
636 xa_destroy(&dev->cqs_xa);
637 ib_dealloc_device(&dev->ibdev);
638 pci_disable_device(pdev);
639 }
640
efa_probe(struct pci_dev * pdev,const struct pci_device_id * ent)641 static int efa_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
642 {
643 struct efa_dev *dev;
644 int err;
645
646 dev = efa_probe_device(pdev);
647 if (IS_ERR(dev))
648 return PTR_ERR(dev);
649
650 err = efa_ib_device_add(dev);
651 if (err)
652 goto err_remove_device;
653
654 return 0;
655
656 err_remove_device:
657 efa_remove_device(pdev);
658 return err;
659 }
660
efa_remove(struct pci_dev * pdev)661 static void efa_remove(struct pci_dev *pdev)
662 {
663 struct efa_dev *dev = pci_get_drvdata(pdev);
664
665 efa_ib_device_remove(dev);
666 efa_remove_device(pdev);
667 }
668
669 static struct pci_driver efa_pci_driver = {
670 .name = DRV_MODULE_NAME,
671 .id_table = efa_pci_tbl,
672 .probe = efa_probe,
673 .remove = efa_remove,
674 };
675
676 module_pci_driver(efa_pci_driver);
677