215 	struct irdma_device *iwdev = rf->iwdev;  in irdma_process_aeq()  local
230 		ibdev_dbg(&iwdev->ibdev,  in irdma_process_aeq()
242 					atomic_dec(&iwdev->vsi.qp_suspend_reqs);  in irdma_process_aeq()
243 					wake_up(&iwdev->suspend_wq);  in irdma_process_aeq()
246 				ibdev_dbg(&iwdev->ibdev, "AEQ: qp_id %d is already freed\n",  in irdma_process_aeq()
313 			if (iwqp->iwdev->vsi.tc_change_pending) {  in irdma_process_aeq()
315 				wake_up(&iwqp->iwdev->suspend_wq);  in irdma_process_aeq()
325 			ibdev_err(&iwdev->ibdev,  in irdma_process_aeq()
370 			ibdev_err(&iwdev->ibdev, "abnormal ae_id = 0x%x bool qp=%d qp_id = %d\n",  in irdma_process_aeq()
372 			if (rdma_protocol_roce(&iwdev->ibdev, 1)) {  in irdma_process_aeq()
1047 static int irdma_alloc_set_mac(struct irdma_device *iwdev)  in irdma_alloc_set_mac()  argument
1051 	status = irdma_alloc_local_mac_entry(iwdev->rf,  in irdma_alloc_set_mac()
1052 					     &iwdev->mac_ip_table_idx);  in irdma_alloc_set_mac()
1054 		status = irdma_add_local_mac_entry(iwdev->rf,  in irdma_alloc_set_mac()
1055 						   (const u8 *)iwdev->netdev->dev_addr,  in irdma_alloc_set_mac()
1056 						   (u8)iwdev->mac_ip_table_idx);  in irdma_alloc_set_mac()
1058 			irdma_del_local_mac_entry(iwdev->rf,  in irdma_alloc_set_mac()
1059 						  (u8)iwdev->mac_ip_table_idx);  in irdma_alloc_set_mac()
1094 		ibdev_dbg(&rf->iwdev->ibdev, "ERR: ceq irq config fail\n");  in irdma_cfg_ceq_vector()
1122 		ibdev_dbg(&rf->iwdev->ibdev, "ERR: aeq irq config fail\n");  in irdma_cfg_aeq_vector()
1213 		ibdev_dbg(&rf->iwdev->ibdev, "ERR: create ceq status = %d\n",  in irdma_setup_ceq_0()
1267 			ibdev_dbg(&rf->iwdev->ibdev,  in irdma_setup_ceqs()
1427 static int irdma_initialize_ilq(struct irdma_device *iwdev)  in irdma_initialize_ilq()  argument
1438 	info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);  in irdma_initialize_ilq()
1444 	status = irdma_puda_create_rsrc(&iwdev->vsi, &info);  in irdma_initialize_ilq()
1446 		ibdev_dbg(&iwdev->ibdev, "ERR: ilq create fail\n");  in irdma_initialize_ilq()
1457 static int irdma_initialize_ieq(struct irdma_device *iwdev)  in irdma_initialize_ieq()  argument
1464 	info.qp_id = iwdev->vsi.exception_lan_q;  in irdma_initialize_ieq()
1468 	info.sq_size = min(iwdev->rf->max_qp / 2, (u32)32768);  in irdma_initialize_ieq()
1470 	info.buf_size = iwdev->vsi.mtu + IRDMA_IPV4_PAD;  in irdma_initialize_ieq()
1472 	status = irdma_puda_create_rsrc(&iwdev->vsi, &info);  in irdma_initialize_ieq()
1474 		ibdev_dbg(&iwdev->ibdev, "ERR: ieq create fail\n");  in irdma_initialize_ieq()
1485 	struct irdma_device *iwdev = vsi->back_vsi;  in irdma_reinitialize_ieq()  local
1486 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_reinitialize_ieq()
1489 	if (irdma_initialize_ieq(iwdev)) {  in irdma_reinitialize_ieq()
1490 		iwdev->rf->reset = true;  in irdma_reinitialize_ieq()
1617 void irdma_rt_deinit_hw(struct irdma_device *iwdev)  in irdma_rt_deinit_hw()  argument
1619 	ibdev_dbg(&iwdev->ibdev, "INIT: state = %d\n", iwdev->init_state);  in irdma_rt_deinit_hw()
1621 	switch (iwdev->init_state) {  in irdma_rt_deinit_hw()
1623 		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_rt_deinit_hw()
1624 			irdma_del_local_mac_entry(iwdev->rf,  in irdma_rt_deinit_hw()
1625 						  (u8)iwdev->mac_ip_table_idx);  in irdma_rt_deinit_hw()
1631 		if (!iwdev->roce_mode)  in irdma_rt_deinit_hw()
1632 			irdma_puda_dele_rsrc(&iwdev->vsi, IRDMA_PUDA_RSRC_TYPE_IEQ,  in irdma_rt_deinit_hw()
1633 					     iwdev->rf->reset);  in irdma_rt_deinit_hw()
1636 		if (!iwdev->roce_mode)  in irdma_rt_deinit_hw()
1637 			irdma_puda_dele_rsrc(&iwdev->vsi,  in irdma_rt_deinit_hw()
1639 					     iwdev->rf->reset);  in irdma_rt_deinit_hw()
1642 		ibdev_warn(&iwdev->ibdev, "bad init_state = %d\n", iwdev->init_state);  in irdma_rt_deinit_hw()
1646 	irdma_cleanup_cm_core(&iwdev->cm_core);  in irdma_rt_deinit_hw()
1647 	if (iwdev->vsi.pestat) {  in irdma_rt_deinit_hw()
1648 		irdma_vsi_stats_free(&iwdev->vsi);  in irdma_rt_deinit_hw()
1649 		kfree(iwdev->vsi.pestat);  in irdma_rt_deinit_hw()
1651 	if (iwdev->cleanup_wq)  in irdma_rt_deinit_hw()
1652 		destroy_workqueue(iwdev->cleanup_wq);  in irdma_rt_deinit_hw()
1695 static void irdma_get_used_rsrc(struct irdma_device *iwdev)  in irdma_get_used_rsrc()  argument
1697 	iwdev->rf->used_pds = find_first_zero_bit(iwdev->rf->allocated_pds,  in irdma_get_used_rsrc()
1698 						 iwdev->rf->max_pd);  in irdma_get_used_rsrc()
1699 	iwdev->rf->used_qps = find_first_zero_bit(iwdev->rf->allocated_qps,  in irdma_get_used_rsrc()
1700 						 iwdev->rf->max_qp);  in irdma_get_used_rsrc()
1701 	iwdev->rf->used_cqs = find_first_zero_bit(iwdev->rf->allocated_cqs,  in irdma_get_used_rsrc()
1702 						 iwdev->rf->max_cq);  in irdma_get_used_rsrc()
1703 	iwdev->rf->used_mrs = find_first_zero_bit(iwdev->rf->allocated_mrs,  in irdma_get_used_rsrc()
1704 						 iwdev->rf->max_mr);  in irdma_get_used_rsrc()
1738 		ibdev_warn(&rf->iwdev->ibdev, "bad init_state = %d\n", rf->init_state);  in irdma_ctrl_deinit_hw()
1751 int irdma_rt_init_hw(struct irdma_device *iwdev,  in irdma_rt_init_hw()  argument
1754 	struct irdma_pci_f *rf = iwdev->rf;  in irdma_rt_init_hw()
1761 	vsi_info.back_vsi = iwdev;  in irdma_rt_init_hw()
1763 	vsi_info.pf_data_vsi_num = iwdev->vsi_num;  in irdma_rt_init_hw()
1767 	irdma_sc_vsi_init(&iwdev->vsi, &vsi_info);  in irdma_rt_init_hw()
1769 	status = irdma_setup_cm_core(iwdev, rf->rdma_ver);  in irdma_rt_init_hw()
1775 		irdma_cleanup_cm_core(&iwdev->cm_core);  in irdma_rt_init_hw()
1779 	status = irdma_vsi_stats_init(&iwdev->vsi, &stats_info);  in irdma_rt_init_hw()
1781 		irdma_cleanup_cm_core(&iwdev->cm_core);  in irdma_rt_init_hw()
1787 		if (!iwdev->roce_mode) {  in irdma_rt_init_hw()
1788 			status = irdma_initialize_ilq(iwdev);  in irdma_rt_init_hw()
1791 			iwdev->init_state = ILQ_CREATED;  in irdma_rt_init_hw()
1792 			status = irdma_initialize_ieq(iwdev);  in irdma_rt_init_hw()
1795 			iwdev->init_state = IEQ_CREATED;  in irdma_rt_init_hw()
1798 			status = irdma_setup_ceqs(rf, &iwdev->vsi);  in irdma_rt_init_hw()
1802 			iwdev->init_state = CEQS_CREATED;  in irdma_rt_init_hw()
1811 			iwdev->init_state = PBLE_CHUNK_MEM;  in irdma_rt_init_hw()
1819 			iwdev->init_state = AEQ_CREATED;  in irdma_rt_init_hw()
1823 		if (iwdev->rf->sc_dev.hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)  in irdma_rt_init_hw()
1824 			irdma_alloc_set_mac(iwdev);  in irdma_rt_init_hw()
1825 		irdma_add_ip(iwdev);  in irdma_rt_init_hw()
1826 		iwdev->init_state = IP_ADDR_REGISTERED;  in irdma_rt_init_hw()
1831 		iwdev->cleanup_wq = alloc_workqueue("irdma-cleanup-wq",  in irdma_rt_init_hw()
1833 		if (!iwdev->cleanup_wq)  in irdma_rt_init_hw()
1835 		irdma_get_used_rsrc(iwdev);  in irdma_rt_init_hw()
1836 		init_waitqueue_head(&iwdev->suspend_wq);  in irdma_rt_init_hw()
1842 		status, iwdev->init_state);  in irdma_rt_init_hw()
1843 	irdma_rt_deinit_hw(iwdev);  in irdma_rt_init_hw()
2052 			ibdev_err(&rf->iwdev->ibdev, "cqp opcode = 0x%x maj_err_code = 0x%x min_err_code = 0x%x\n",  in irdma_cqp_ce_handler()
2141 	irdma_hw_modify_qp(iwqp->iwdev, iwqp, &info, 0);  in irdma_next_iw_state()
2248 static int irdma_cqp_manage_apbvt_cmd(struct irdma_device *iwdev,  in irdma_cqp_manage_apbvt_cmd()  argument
2256 	cqp_request = irdma_alloc_and_get_cqp_request(&iwdev->rf->cqp, add_port);  in irdma_cqp_manage_apbvt_cmd()
2267 	cqp_info->in.u.manage_apbvt_entry.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_cqp_manage_apbvt_cmd()
2269 	ibdev_dbg(&iwdev->ibdev, "DEV: %s: port=0x%04x\n",  in irdma_cqp_manage_apbvt_cmd()
2272 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_cqp_manage_apbvt_cmd()
2273 	irdma_put_cqp_request(&iwdev->rf->cqp, cqp_request);  in irdma_cqp_manage_apbvt_cmd()
2283 struct irdma_apbvt_entry *irdma_add_apbvt(struct irdma_device *iwdev, u16 port)  in irdma_add_apbvt()  argument
2285 	struct irdma_cm_core *cm_core = &iwdev->cm_core;  in irdma_add_apbvt()
2307 	if (irdma_cqp_manage_apbvt_cmd(iwdev, port, true)) {  in irdma_add_apbvt()
2320 void irdma_del_apbvt(struct irdma_device *iwdev,  in irdma_del_apbvt()  argument
2323 	struct irdma_cm_core *cm_core = &iwdev->cm_core;  in irdma_del_apbvt()
2337 	irdma_cqp_manage_apbvt_cmd(iwdev, entry->port, false);  in irdma_del_apbvt()
2412 int irdma_manage_qhash(struct irdma_device *iwdev, struct irdma_cm_info *cminfo,  in irdma_manage_qhash()  argument
2418 	struct irdma_cqp *iwcqp = &iwdev->rf->cqp;  in irdma_manage_qhash()
2431 	info->vsi = &iwdev->vsi;  in irdma_manage_qhash()
2442 	ether_addr_copy(info->mac_addr, iwdev->netdev->dev_addr);  in irdma_manage_qhash()
2467 		ibdev_dbg(&iwdev->ibdev,  in irdma_manage_qhash()
2475 		ibdev_dbg(&iwdev->ibdev,  in irdma_manage_qhash()
2483 	cqp_info->in.u.manage_qhash_table_entry.cqp = &iwdev->rf->cqp.sc_cqp;  in irdma_manage_qhash()
2487 	status = irdma_handle_cqp_op(iwdev->rf, cqp_request);  in irdma_manage_qhash()
2525 			ibdev_err(&iwqp->iwdev->ibdev, "Flush QP[%d] failed, SQ has more work",  in irdma_hw_flush_wqes_callback()
2614 					ibdev_err(&iwqp->iwdev->ibdev, "fatal QP event: SQ in error but not flushed, qp: %d",  in irdma_hw_flush_wqes()
2630 	ibdev_dbg(&rf->iwdev->ibdev,  in irdma_hw_flush_wqes()
2675 	struct irdma_pci_f *rf = iwqp->iwdev->rf;  in irdma_flush_wqes()
2705 			queue_delayed_work(iwqp->iwdev->cleanup_wq,  in irdma_flush_wqes()