738 	struct irdma_qp *iwqp = (struct irdma_qp *)ibqp;  in irdma_qp_add_ref()  local
740 	refcount_inc(&iwqp->refcnt);  in irdma_qp_add_ref()
745 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_qp_rem_ref()  local
746 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_qp_rem_ref()
751 	if (!refcount_dec_and_test(&iwqp->refcnt)) {  in irdma_qp_rem_ref()
756 	qp_num = iwqp->ibqp.qp_num;  in irdma_qp_rem_ref()
759 	complete(&iwqp->free_qp);  in irdma_qp_rem_ref()
872 	struct irdma_qp *iwqp;  in irdma_term_modify_qp()  local
874 	iwqp = qp->qp_uk.back_qp;  in irdma_term_modify_qp()
875 	irdma_next_iw_state(iwqp, next_state, 0, term, term_len);  in irdma_term_modify_qp()
885 	struct irdma_qp *iwqp;  in irdma_terminate_done()  local
890 	iwqp = qp->qp_uk.back_qp;  in irdma_terminate_done()
891 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_terminate_done()
892 	if (iwqp->hte_added) {  in irdma_terminate_done()
893 		iwqp->hte_added = 0;  in irdma_terminate_done()
898 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_terminate_done()
903 		irdma_next_iw_state(iwqp, IRDMA_QP_STATE_ERROR, hte, 0, 0);  in irdma_terminate_done()
904 		irdma_cm_disconn(iwqp);  in irdma_terminate_done()
910 	struct irdma_qp *iwqp = from_timer(iwqp, t, terminate_timer);  in irdma_terminate_timeout()  local
911 	struct irdma_sc_qp *qp = &iwqp->sc_qp;  in irdma_terminate_timeout()
914 	irdma_qp_rem_ref(&iwqp->ibqp);  in irdma_terminate_timeout()
923 	struct irdma_qp *iwqp;  in irdma_terminate_start_timer()  local
925 	iwqp = qp->qp_uk.back_qp;  in irdma_terminate_start_timer()
926 	irdma_qp_add_ref(&iwqp->ibqp);  in irdma_terminate_start_timer()
927 	timer_setup(&iwqp->terminate_timer, irdma_terminate_timeout, 0);  in irdma_terminate_start_timer()
928 	iwqp->terminate_timer.expires = jiffies + HZ;  in irdma_terminate_start_timer()
930 	add_timer(&iwqp->terminate_timer);  in irdma_terminate_start_timer()
939 	struct irdma_qp *iwqp;  in irdma_terminate_del_timer()  local
942 	iwqp = qp->qp_uk.back_qp;  in irdma_terminate_del_timer()
943 	ret = del_timer(&iwqp->terminate_timer);  in irdma_terminate_del_timer()
945 		irdma_qp_rem_ref(&iwqp->ibqp);  in irdma_terminate_del_timer()
1117 void irdma_free_qp_rsrc(struct irdma_qp *iwqp)  in irdma_free_qp_rsrc()  argument
1119 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_free_qp_rsrc()
1121 	u32 qp_num = iwqp->ibqp.qp_num;  in irdma_free_qp_rsrc()
1123 	irdma_ieq_cleanup_qp(iwdev->vsi.ieq, &iwqp->sc_qp);  in irdma_free_qp_rsrc()
1124 	irdma_dealloc_push_page(rf, &iwqp->sc_qp);  in irdma_free_qp_rsrc()
1125 	if (iwqp->sc_qp.vsi) {  in irdma_free_qp_rsrc()
1126 		irdma_qp_rem_qos(&iwqp->sc_qp);  in irdma_free_qp_rsrc()
1127 		iwqp->sc_qp.dev->ws_remove(iwqp->sc_qp.vsi,  in irdma_free_qp_rsrc()
1128 					   iwqp->sc_qp.user_pri);  in irdma_free_qp_rsrc()
1133 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->q2_ctx_mem.size,  in irdma_free_qp_rsrc()
1134 			  iwqp->q2_ctx_mem.va, iwqp->q2_ctx_mem.pa);  in irdma_free_qp_rsrc()
1135 	iwqp->q2_ctx_mem.va = NULL;  in irdma_free_qp_rsrc()
1136 	dma_free_coherent(rf->sc_dev.hw->device, iwqp->kqp.dma_mem.size,  in irdma_free_qp_rsrc()
1137 			  iwqp->kqp.dma_mem.va, iwqp->kqp.dma_mem.pa);  in irdma_free_qp_rsrc()
1138 	iwqp->kqp.dma_mem.va = NULL;  in irdma_free_qp_rsrc()
1139 	kfree(iwqp->kqp.sq_wrid_mem);  in irdma_free_qp_rsrc()
1140 	kfree(iwqp->kqp.rq_wrid_mem);  in irdma_free_qp_rsrc()
1174 	struct irdma_qp *iwqp;  in irdma_hw_modify_qp_callback()  local
1177 	iwqp = cqp_info->in.u.qp_modify.qp->qp_uk.back_qp;  in irdma_hw_modify_qp_callback()
1178 	atomic_dec(&iwqp->hw_mod_qp_pend);  in irdma_hw_modify_qp_callback()
1179 	wake_up(&iwqp->mod_qp_waitq);  in irdma_hw_modify_qp_callback()
1189 int irdma_hw_modify_qp(struct irdma_device *iwdev, struct irdma_qp *iwqp,  in irdma_hw_modify_qp()  argument
1204 		atomic_inc(&iwqp->hw_mod_qp_pend);  in irdma_hw_modify_qp()
1211 	cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;  in irdma_hw_modify_qp()
1227 				irdma_send_reset(iwqp->cm_node);  in irdma_hw_modify_qp()
1229 				iwqp->sc_qp.term_flags = IRDMA_TERM_DONE;  in irdma_hw_modify_qp()
1233 				irdma_gen_ae(rf, &iwqp->sc_qp, &ae_info, false);  in irdma_hw_modify_qp()
1245 				cqp_info->in.u.qp_modify.qp = &iwqp->sc_qp;  in irdma_hw_modify_qp()
1391 	struct irdma_qp *iwqp;  in irdma_ieq_get_qp()  local
1416 	iwqp = cm_node->iwqp;  in irdma_ieq_get_qp()
1419 	return &iwqp->sc_qp;  in irdma_ieq_get_qp()
2467 void irdma_ib_qp_event(struct irdma_qp *iwqp, enum irdma_qp_event_type event)  in irdma_ib_qp_event()  argument
2471 	if (!iwqp->ibqp.event_handler)  in irdma_ib_qp_event()
2482 	ibevent.device = iwqp->ibqp.device;  in irdma_ib_qp_event()
2483 	ibevent.element.qp = &iwqp->ibqp;  in irdma_ib_qp_event()
2484 	iwqp->ibqp.event_handler(&ibevent, iwqp->ibqp.qp_context);  in irdma_ib_qp_event()
2558 void irdma_generate_flush_completions(struct irdma_qp *iwqp)  in irdma_generate_flush_completions()  argument
2560 	struct irdma_qp_uk *qp = &iwqp->sc_qp.qp_uk;  in irdma_generate_flush_completions()
2570 	spin_lock_irqsave(&iwqp->iwscq->lock, flags1);  in irdma_generate_flush_completions()
2571 	if (irdma_cq_empty(iwqp->iwscq)) {  in irdma_generate_flush_completions()
2574 		spin_lock_irqsave(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2578 				spin_unlock_irqrestore(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2579 				spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);  in irdma_generate_flush_completions()
2594 			ibdev_dbg(iwqp->iwscq->ibcq.device,  in irdma_generate_flush_completions()
2597 			list_add_tail(&cmpl->list, &iwqp->iwscq->cmpl_generated);  in irdma_generate_flush_completions()
2600 		spin_unlock_irqrestore(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2601 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);  in irdma_generate_flush_completions()
2603 			irdma_comp_handler(iwqp->iwscq);  in irdma_generate_flush_completions()
2605 		spin_unlock_irqrestore(&iwqp->iwscq->lock, flags1);  in irdma_generate_flush_completions()
2606 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_generate_flush_completions()
2610 	spin_lock_irqsave(&iwqp->iwrcq->lock, flags1);  in irdma_generate_flush_completions()
2611 	if (irdma_cq_empty(iwqp->iwrcq)) {  in irdma_generate_flush_completions()
2614 		spin_lock_irqsave(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2618 				spin_unlock_irqrestore(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2619 				spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);  in irdma_generate_flush_completions()
2630 			ibdev_dbg(iwqp->iwrcq->ibcq.device,  in irdma_generate_flush_completions()
2634 			list_add_tail(&cmpl->list, &iwqp->iwrcq->cmpl_generated);  in irdma_generate_flush_completions()
2638 		spin_unlock_irqrestore(&iwqp->lock, flags2);  in irdma_generate_flush_completions()
2639 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);  in irdma_generate_flush_completions()
2641 			irdma_comp_handler(iwqp->iwrcq);  in irdma_generate_flush_completions()
2643 		spin_unlock_irqrestore(&iwqp->iwrcq->lock, flags1);  in irdma_generate_flush_completions()
2644 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_generate_flush_completions()