23 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_pf_regs()  local
27 	gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_pf_regs()
28 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_pf_regs()
29 				mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF);  in mana_gd_init_pf_regs()
31 	sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF);  in mana_gd_init_pf_regs()
33 	sriov_base_va = gc->bar0_va + sriov_base_off;  in mana_gd_init_pf_regs()
34 	gc->shm_base = sriov_base_va +  in mana_gd_init_pf_regs()
35 			mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF);  in mana_gd_init_pf_regs()
40 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_vf_regs()  local
42 	gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF;  in mana_gd_init_vf_regs()
44 	gc->db_page_base = gc->bar0_va +  in mana_gd_init_vf_regs()
45 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);  in mana_gd_init_vf_regs()
47 	gc->phys_db_page_base = gc->bar0_pa +  in mana_gd_init_vf_regs()
48 				mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET);  in mana_gd_init_vf_regs()
50 	gc->shm_base = gc->bar0_va + mana_gd_r64(gc, GDMA_REG_SHM_OFFSET);  in mana_gd_init_vf_regs()
55 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_init_registers()  local
57 	if (gc->is_pf)  in mana_gd_init_registers()
65 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_query_max_resources()  local
73 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_query_max_resources()
75 		dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n",  in mana_gd_query_max_resources()
80 	if (gc->num_msix_usable > resp.max_msix)  in mana_gd_query_max_resources()
81 		gc->num_msix_usable = resp.max_msix;  in mana_gd_query_max_resources()
83 	if (gc->num_msix_usable <= 1)  in mana_gd_query_max_resources()
86 	gc->max_num_queues = num_online_cpus();  in mana_gd_query_max_resources()
87 	if (gc->max_num_queues > MANA_MAX_NUM_QUEUES)  in mana_gd_query_max_resources()
88 		gc->max_num_queues = MANA_MAX_NUM_QUEUES;  in mana_gd_query_max_resources()
90 	if (gc->max_num_queues > resp.max_eq)  in mana_gd_query_max_resources()
91 		gc->max_num_queues = resp.max_eq;  in mana_gd_query_max_resources()
93 	if (gc->max_num_queues > resp.max_cq)  in mana_gd_query_max_resources()
94 		gc->max_num_queues = resp.max_cq;  in mana_gd_query_max_resources()
96 	if (gc->max_num_queues > resp.max_sq)  in mana_gd_query_max_resources()
97 		gc->max_num_queues = resp.max_sq;  in mana_gd_query_max_resources()
99 	if (gc->max_num_queues > resp.max_rq)  in mana_gd_query_max_resources()
100 		gc->max_num_queues = resp.max_rq;  in mana_gd_query_max_resources()
103 	if (gc->max_num_queues > gc->num_msix_usable - 1)  in mana_gd_query_max_resources()
104 		gc->max_num_queues = gc->num_msix_usable - 1;  in mana_gd_query_max_resources()
111 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_query_hwc_timeout()  local
119 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_query_hwc_timeout()
130 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_detect_devices()  local
141 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_detect_devices()
143 		dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err,  in mana_gd_detect_devices()
159 			gc->mana.gdma_context = gc;  in mana_gd_detect_devices()
160 			gc->mana.dev_id = dev;  in mana_gd_detect_devices()
164 	return gc->mana.dev_id.type == 0 ? -ENODEV : 0;  in mana_gd_detect_devices()
167 int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req,  in mana_gd_send_request()  argument
170 	struct hw_channel_context *hwc = gc->hwc.driver_data;  in mana_gd_send_request()
176 int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length,  in mana_gd_alloc_memory()  argument
185 	gmi->dev = gc->dev;  in mana_gd_alloc_memory()
203 static int mana_gd_create_hw_eq(struct gdma_context *gc,  in mana_gd_create_hw_eq()  argument
225 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_create_hw_eq()
227 		dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err,  in mana_gd_create_hw_eq()
240 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_disable_queue()  local
255 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_disable_queue()
257 		dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err,  in mana_gd_disable_queue()
270 static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index,  in mana_gd_ring_doorbell()  argument
274 	void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index;  in mana_gd_ring_doorbell()
320 void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_wq_ring_doorbell()  argument
325 	mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type,  in mana_gd_wq_ring_doorbell()
331 	struct gdma_context *gc = cq->gdma_dev->gdma_context;  in mana_gd_ring_cq()  local
337 	mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id,  in mana_gd_ring_cq()
344 	struct gdma_context *gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eqe()  local
360 		if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs))  in mana_gd_process_eqe()
363 		cq = gc->cq_table[cq_id];  in mana_gd_process_eqe()
373 		gc->test_event_eq_id = eq->id;  in mana_gd_process_eqe()
374 		complete(&gc->eq_test_event);  in mana_gd_process_eqe()
399 	struct gdma_context *gc;  in mana_gd_process_eq_events()  local
404 	gc = eq->gdma_dev->gdma_context;  in mana_gd_process_eq_events()
422 			dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id);  in mana_gd_process_eq_events()
438 	mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id,  in mana_gd_process_eq_events()
447 	struct gdma_context *gc;  in mana_gd_register_irq()  local
454 	gc = gd->gdma_context;  in mana_gd_register_irq()
455 	r = &gc->msix_resource;  in mana_gd_register_irq()
456 	dev = gc->dev;  in mana_gd_register_irq()
461 	if (msi_index >= r->size || msi_index >= gc->num_msix_usable) {  in mana_gd_register_irq()
472 			err, msi_index, r->size, gc->num_msix_usable);  in mana_gd_register_irq()
477 	gic = &gc->irq_contexts[msi_index];  in mana_gd_register_irq()
492 	struct gdma_context *gc;  in mana_gd_deregiser_irq()  local
497 	gc = gd->gdma_context;  in mana_gd_deregiser_irq()
498 	r = &gc->msix_resource;  in mana_gd_deregiser_irq()
502 	if (WARN_ON(msix_index >= gc->num_msix_usable))  in mana_gd_deregiser_irq()
505 	gic = &gc->irq_contexts[msix_index];  in mana_gd_deregiser_irq()
516 int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq)  in mana_gd_test_eq()  argument
520 	struct device *dev = gc->dev;  in mana_gd_test_eq()
523 	mutex_lock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
525 	init_completion(&gc->eq_test_event);  in mana_gd_test_eq()
526 	gc->test_event_eq_id = INVALID_QUEUE_ID;  in mana_gd_test_eq()
534 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_test_eq()
547 	if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) {  in mana_gd_test_eq()
552 	if (eq->id != gc->test_event_eq_id) {  in mana_gd_test_eq()
554 			gc->test_event_eq_id, eq->id);  in mana_gd_test_eq()
560 	mutex_unlock(&gc->eq_test_event_mutex);  in mana_gd_test_eq()
564 static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets,  in mana_gd_destroy_eq()  argument
570 		err = mana_gd_test_eq(gc, queue);  in mana_gd_destroy_eq()
572 			dev_warn(gc->dev, "Failed to flush EQ: %d\n", err);  in mana_gd_destroy_eq()
585 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_eq()  local
586 	struct device *dev = gc->dev;  in mana_gd_create_eq()
612 		err = mana_gd_create_hw_eq(gc, queue);  in mana_gd_create_eq()
616 		err = mana_gd_test_eq(gc, queue);  in mana_gd_create_eq()
624 	mana_gd_destroy_eq(gc, false, queue);  in mana_gd_create_eq()
639 static void mana_gd_destroy_cq(struct gdma_context *gc,  in mana_gd_destroy_cq()  argument
644 	if (id >= gc->max_num_cqs)  in mana_gd_destroy_cq()
647 	if (!gc->cq_table[id])  in mana_gd_destroy_cq()
650 	gc->cq_table[id] = NULL;  in mana_gd_destroy_cq()
657 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_hwc_queue()  local
667 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_hwc_queue()
696 int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle)  in mana_gd_destroy_dma_region()  argument
709 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_destroy_dma_region()
711 		dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n",  in mana_gd_destroy_dma_region()
726 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_dma_region()  local
739 	hwc = gc->hwc.driver_data;  in mana_gd_create_dma_region()
759 	err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp);  in mana_gd_create_dma_region()
765 		dev_err(gc->dev, "Failed to create DMA region: 0x%x\n",  in mana_gd_create_dma_region()
781 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_eq()  local
794 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_eq()
827 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_create_mana_wq_cq()  local
841 	err = mana_gd_alloc_memory(gc, spec->queue_size, gmi);  in mana_gd_create_mana_wq_cq()
869 void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue)  in mana_gd_destroy_queue()  argument
875 		mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue);  in mana_gd_destroy_queue()
879 		mana_gd_destroy_cq(gc, queue);  in mana_gd_destroy_queue()
889 		dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n",  in mana_gd_destroy_queue()
894 	mana_gd_destroy_dma_region(gc, gmi->dma_region_handle);  in mana_gd_destroy_queue()
901 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_verify_vf_version()  local
907 	hwc = gc->hwc.driver_data;  in mana_gd_verify_vf_version()
928 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_verify_vf_version()
930 		dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n",  in mana_gd_verify_vf_version()
937 			dev_err(gc->dev, "Failed to set the hwc timeout %d\n", err);  in mana_gd_verify_vf_version()
940 		dev_dbg(gc->dev, "set the hwc timeout to %u\n", hwc->hwc_timeout);  in mana_gd_verify_vf_version()
947 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_register_device()  local
961 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_register_device()
963 		dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n",  in mana_gd_register_device()
977 	struct gdma_context *gc = gd->gdma_context;  in mana_gd_deregister_device()  local
990 	err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp);  in mana_gd_deregister_device()
992 		dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n",  in mana_gd_deregister_device()
1095 	struct gdma_context *gc;  in mana_gd_post_work_request()  local
1126 		gc = wq->gdma_dev->gdma_context;  in mana_gd_post_work_request()
1127 		dev_err(gc->dev, "unsuccessful flow control!\n");  in mana_gd_post_work_request()
1151 	struct gdma_context *gc = queue->gdma_dev->gdma_context;  in mana_gd_post_and_ring()  local
1158 	mana_gd_wq_ring_doorbell(gc, queue);  in mana_gd_post_and_ring()
1249 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup_irqs()  local
1265 	gc->irq_contexts = kcalloc(nvec, sizeof(struct gdma_irq_context),  in mana_gd_setup_irqs()
1267 	if (!gc->irq_contexts) {  in mana_gd_setup_irqs()
1273 		gic = &gc->irq_contexts[i];  in mana_gd_setup_irqs()
1294 		cpu = cpumask_local_spread(i, gc->numa_node);  in mana_gd_setup_irqs()
1298 	err = mana_gd_alloc_res_map(nvec, &gc->msix_resource);  in mana_gd_setup_irqs()
1302 	gc->max_num_msix = nvec;  in mana_gd_setup_irqs()
1303 	gc->num_msix_usable = nvec;  in mana_gd_setup_irqs()
1310 		gic = &gc->irq_contexts[j];  in mana_gd_setup_irqs()
1316 	kfree(gc->irq_contexts);  in mana_gd_setup_irqs()
1317 	gc->irq_contexts = NULL;  in mana_gd_setup_irqs()
1325 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove_irqs()  local
1329 	if (gc->max_num_msix < 1)  in mana_gd_remove_irqs()
1332 	mana_gd_free_res_map(&gc->msix_resource);  in mana_gd_remove_irqs()
1334 	for (i = 0; i < gc->max_num_msix; i++) {  in mana_gd_remove_irqs()
1339 		gic = &gc->irq_contexts[i];  in mana_gd_remove_irqs()
1348 	gc->max_num_msix = 0;  in mana_gd_remove_irqs()
1349 	gc->num_msix_usable = 0;  in mana_gd_remove_irqs()
1350 	kfree(gc->irq_contexts);  in mana_gd_remove_irqs()
1351 	gc->irq_contexts = NULL;  in mana_gd_remove_irqs()
1356 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_setup()  local
1360 	mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base);  in mana_gd_setup()
1366 	err = mana_hwc_create_channel(gc);  in mana_gd_setup()
1385 	mana_hwc_destroy_channel(gc);  in mana_gd_setup()
1393 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_cleanup()  local
1395 	mana_hwc_destroy_channel(gc);  in mana_gd_cleanup()
1407 	struct gdma_context *gc;  in mana_gd_probe()  local
1436 	gc = vzalloc(sizeof(*gc));  in mana_gd_probe()
1437 	if (!gc)  in mana_gd_probe()
1440 	mutex_init(&gc->eq_test_event_mutex);  in mana_gd_probe()
1441 	pci_set_drvdata(pdev, gc);  in mana_gd_probe()
1442 	gc->bar0_pa = pci_resource_start(pdev, 0);  in mana_gd_probe()
1448 	gc->numa_node = dev_to_node(&pdev->dev);  in mana_gd_probe()
1449 	gc->is_pf = mana_is_pf(pdev->device);  in mana_gd_probe()
1450 	gc->bar0_va = bar0_va;  in mana_gd_probe()
1451 	gc->dev = &pdev->dev;  in mana_gd_probe()
1457 	err = mana_probe(&gc->mana, false);  in mana_gd_probe()
1469 	vfree(gc);  in mana_gd_probe()
1480 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_remove()  local
1482 	mana_remove(&gc->mana, false);  in mana_gd_remove()
1486 	pci_iounmap(pdev, gc->bar0_va);  in mana_gd_remove()
1488 	vfree(gc);  in mana_gd_remove()
1497 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_suspend()  local
1499 	mana_remove(&gc->mana, true);  in mana_gd_suspend()
1512 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_resume()  local
1519 	err = mana_probe(&gc->mana, true);  in mana_gd_resume()
1529 	struct gdma_context *gc = pci_get_drvdata(pdev);  in mana_gd_shutdown()  local
1533 	mana_remove(&gc->mana, true);  in mana_gd_shutdown()