258 static void irdma_alloc_push_page(struct irdma_qp *iwqp)  in irdma_alloc_push_page()  argument
262 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_alloc_push_page()
263 	struct irdma_sc_qp *qp = &iwqp->sc_qp;  in irdma_alloc_push_page()
471 static void irdma_clean_cqes(struct irdma_qp *iwqp, struct irdma_cq *iwcq)  in irdma_clean_cqes()  argument
477 	irdma_uk_clean_cq(&iwqp->sc_qp.qp_uk, ukcq);  in irdma_clean_cqes()
481 static void irdma_remove_push_mmap_entries(struct irdma_qp *iwqp)  in irdma_remove_push_mmap_entries()  argument
483 	if (iwqp->push_db_mmap_entry) {  in irdma_remove_push_mmap_entries()
484 		rdma_user_mmap_entry_remove(iwqp->push_db_mmap_entry);  in irdma_remove_push_mmap_entries()
485 		iwqp->push_db_mmap_entry = NULL;  in irdma_remove_push_mmap_entries()
487 	if (iwqp->push_wqe_mmap_entry) {  in irdma_remove_push_mmap_entries()
488 		rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);  in irdma_remove_push_mmap_entries()
489 		iwqp->push_wqe_mmap_entry = NULL;  in irdma_remove_push_mmap_entries()
494 					 struct irdma_qp *iwqp,  in irdma_setup_push_mmap_entries()  argument
506 	bar_off += rsvd + iwqp->sc_qp.push_idx * IRDMA_HW_PAGE_SIZE;  in irdma_setup_push_mmap_entries()
507 	iwqp->push_wqe_mmap_entry = irdma_user_mmap_entry_insert(ucontext,  in irdma_setup_push_mmap_entries()
510 	if (!iwqp->push_wqe_mmap_entry)  in irdma_setup_push_mmap_entries()
515 	iwqp->push_db_mmap_entry = irdma_user_mmap_entry_insert(ucontext,  in irdma_setup_push_mmap_entries()
518 	if (!iwqp->push_db_mmap_entry) {  in irdma_setup_push_mmap_entries()
519 		rdma_user_mmap_entry_remove(iwqp->push_wqe_mmap_entry);  in irdma_setup_push_mmap_entries()
533 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_destroy_qp()  local
534 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_destroy_qp()
536 	iwqp->sc_qp.qp_uk.destroy_pending = true;  in irdma_destroy_qp()
538 	if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS)  in irdma_destroy_qp()
539 		irdma_modify_qp_to_err(&iwqp->sc_qp);  in irdma_destroy_qp()
541 	if (!iwqp->user_mode)  in irdma_destroy_qp()
542 		cancel_delayed_work_sync(&iwqp->dwork_flush);  in irdma_destroy_qp()
544 	irdma_qp_rem_ref(&iwqp->ibqp);  in irdma_destroy_qp()
545 	wait_for_completion(&iwqp->free_qp);  in irdma_destroy_qp()
546 	irdma_free_lsmm_rsrc(iwqp);  in irdma_destroy_qp()
547 	irdma_cqp_qp_destroy_cmd(&iwdev->rf->sc_dev, &iwqp->sc_qp);  in irdma_destroy_qp()
549 	if (!iwqp->user_mode) {  in irdma_destroy_qp()
550 		if (iwqp->iwscq) {  in irdma_destroy_qp()
551 			irdma_clean_cqes(iwqp, iwqp->iwscq);  in irdma_destroy_qp()
552 			if (iwqp->iwrcq != iwqp->iwscq)  in irdma_destroy_qp()
553 				irdma_clean_cqes(iwqp, iwqp->iwrcq);  in irdma_destroy_qp()
556 	irdma_remove_push_mmap_entries(iwqp);  in irdma_destroy_qp()
557 	irdma_free_qp_rsrc(iwqp);  in irdma_destroy_qp()
569 			       struct irdma_qp *iwqp,  in irdma_setup_virt_qp()  argument
572 	struct irdma_pbl *iwpbl = iwqp->iwpbl;  in irdma_setup_virt_qp()
575 	iwqp->page = qpmr->sq_page;  in irdma_setup_virt_qp()
595 				struct irdma_qp *iwqp,  in irdma_setup_kmode_qp()  argument
599 	struct irdma_dma_mem *mem = &iwqp->kqp.dma_mem;  in irdma_setup_kmode_qp()
627 	iwqp->kqp.sq_wrid_mem =  in irdma_setup_kmode_qp()
628 		kcalloc(sqdepth, sizeof(*iwqp->kqp.sq_wrid_mem), GFP_KERNEL);  in irdma_setup_kmode_qp()
629 	if (!iwqp->kqp.sq_wrid_mem)  in irdma_setup_kmode_qp()
632 	iwqp->kqp.rq_wrid_mem =  in irdma_setup_kmode_qp()
633 		kcalloc(rqdepth, sizeof(*iwqp->kqp.rq_wrid_mem), GFP_KERNEL);  in irdma_setup_kmode_qp()
634 	if (!iwqp->kqp.rq_wrid_mem) {  in irdma_setup_kmode_qp()
635 		kfree(iwqp->kqp.sq_wrid_mem);  in irdma_setup_kmode_qp()
636 		iwqp->kqp.sq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
640 	ukinfo->sq_wrtrk_array = iwqp->kqp.sq_wrid_mem;  in irdma_setup_kmode_qp()
641 	ukinfo->rq_wrid_array = iwqp->kqp.rq_wrid_mem;  in irdma_setup_kmode_qp()
650 		kfree(iwqp->kqp.sq_wrid_mem);  in irdma_setup_kmode_qp()
651 		iwqp->kqp.sq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
652 		kfree(iwqp->kqp.rq_wrid_mem);  in irdma_setup_kmode_qp()
653 		iwqp->kqp.rq_wrid_mem = NULL;  in irdma_setup_kmode_qp()
665 	ukinfo->qp_id = iwqp->ibqp.qp_num;  in irdma_setup_kmode_qp()
673 static int irdma_cqp_create_qp_cmd(struct irdma_qp *iwqp)  in irdma_cqp_create_qp_cmd()  argument
675 	struct irdma_pci_f *rf = iwqp->iwdev->rf;  in irdma_cqp_create_qp_cmd()
694 	cqp_info->in.u.qp_create.qp = &iwqp->sc_qp;  in irdma_cqp_create_qp_cmd()
702 static void irdma_roce_fill_and_set_qpctx_info(struct irdma_qp *iwqp,  in irdma_roce_fill_and_set_qpctx_info()  argument
705 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_roce_fill_and_set_qpctx_info()
710 	udp_info = &iwqp->udp_info;  in irdma_roce_fill_and_set_qpctx_info()
717 	roce_info = &iwqp->roce_info;  in irdma_roce_fill_and_set_qpctx_info()
730 	if (!iwqp->user_mode) {  in irdma_roce_fill_and_set_qpctx_info()
737 	ctx_info->roce_info = &iwqp->roce_info;  in irdma_roce_fill_and_set_qpctx_info()
738 	ctx_info->udp_info = &iwqp->udp_info;  in irdma_roce_fill_and_set_qpctx_info()
739 	irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_roce_fill_and_set_qpctx_info()
742 static void irdma_iw_fill_and_set_qpctx_info(struct irdma_qp *iwqp,  in irdma_iw_fill_and_set_qpctx_info()  argument
745 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_iw_fill_and_set_qpctx_info()
749 	iwarp_info = &iwqp->iwarp_info;  in irdma_iw_fill_and_set_qpctx_info()
759 	if (!iwqp->user_mode) {  in irdma_iw_fill_and_set_qpctx_info()
766 	ctx_info->iwarp_info = &iwqp->iwarp_info;  in irdma_iw_fill_and_set_qpctx_info()
768 	irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_iw_fill_and_set_qpctx_info()
802 	struct irdma_qp *iwqp = container_of(dwork, struct irdma_qp, dwork_flush);  in irdma_flush_worker()  local
804 	irdma_generate_flush_completions(iwqp);  in irdma_flush_worker()
821 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_create_qp()  local
850 	qp = &iwqp->sc_qp;  in irdma_create_qp()
851 	qp->qp_uk.back_qp = iwqp;  in irdma_create_qp()
854 	iwqp->iwdev = iwdev;  in irdma_create_qp()
855 	iwqp->q2_ctx_mem.size = ALIGN(IRDMA_Q2_BUF_SIZE + IRDMA_QP_CTX_SIZE,  in irdma_create_qp()
857 	iwqp->q2_ctx_mem.va = dma_alloc_coherent(dev->hw->device,  in irdma_create_qp()
858 						 iwqp->q2_ctx_mem.size,  in irdma_create_qp()
859 						 &iwqp->q2_ctx_mem.pa,  in irdma_create_qp()
861 	if (!iwqp->q2_ctx_mem.va)  in irdma_create_qp()
864 	init_info.q2 = iwqp->q2_ctx_mem.va;  in irdma_create_qp()
865 	init_info.q2_pa = iwqp->q2_ctx_mem.pa;  in irdma_create_qp()
877 	iwqp->iwpd = iwpd;  in irdma_create_qp()
878 	iwqp->ibqp.qp_num = qp_num;  in irdma_create_qp()
879 	qp = &iwqp->sc_qp;  in irdma_create_qp()
880 	iwqp->iwscq = to_iwcq(init_attr->send_cq);  in irdma_create_qp()
881 	iwqp->iwrcq = to_iwcq(init_attr->recv_cq);  in irdma_create_qp()
882 	iwqp->host_ctx.va = init_info.host_ctx;  in irdma_create_qp()
883 	iwqp->host_ctx.pa = init_info.host_ctx_pa;  in irdma_create_qp()
884 	iwqp->host_ctx.size = IRDMA_QP_CTX_SIZE;  in irdma_create_qp()
887 	init_info.qp_uk_init_info.qp_id = iwqp->ibqp.qp_num;  in irdma_create_qp()
890 	iwqp->ctx_info.qp_compl_ctx = (uintptr_t)qp;  in irdma_create_qp()
891 	init_waitqueue_head(&iwqp->waitq);  in irdma_create_qp()
892 	init_waitqueue_head(&iwqp->mod_qp_waitq);  in irdma_create_qp()
903 		iwqp->ctx_info.qp_compl_ctx = req.user_compl_ctx;  in irdma_create_qp()
904 		iwqp->user_mode = 1;  in irdma_create_qp()
913 			iwqp->iwpbl = irdma_get_pbl((unsigned long)req.user_wqe_bufs,  in irdma_create_qp()
917 			if (!iwqp->iwpbl) {  in irdma_create_qp()
924 		irdma_setup_virt_qp(iwdev, iwqp, &init_info);  in irdma_create_qp()
926 		INIT_DELAYED_WORK(&iwqp->dwork_flush, irdma_flush_worker);  in irdma_create_qp()
928 		err_code = irdma_setup_kmode_qp(iwdev, iwqp, &init_info, init_attr);  in irdma_create_qp()
961 	ctx_info = &iwqp->ctx_info;  in irdma_create_qp()
962 	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_create_qp()
963 	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_create_qp()
966 		irdma_roce_fill_and_set_qpctx_info(iwqp, ctx_info);  in irdma_create_qp()
968 		irdma_iw_fill_and_set_qpctx_info(iwqp, ctx_info);  in irdma_create_qp()
970 	err_code = irdma_cqp_create_qp_cmd(iwqp);  in irdma_create_qp()
974 	refcount_set(&iwqp->refcnt, 1);  in irdma_create_qp()
975 	spin_lock_init(&iwqp->lock);  in irdma_create_qp()
976 	spin_lock_init(&iwqp->sc_qp.pfpdu.lock);  in irdma_create_qp()
977 	iwqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR) ? 1 : 0;  in irdma_create_qp()
978 	rf->qp_table[qp_num] = iwqp;  in irdma_create_qp()
979 	iwqp->max_send_wr = sq_size;  in irdma_create_qp()
980 	iwqp->max_recv_wr = rq_size;  in irdma_create_qp()
984 			irdma_cqp_qp_destroy_cmd(&rf->sc_dev, &iwqp->sc_qp);  in irdma_create_qp()
989 		irdma_qp_add_qos(&iwqp->sc_qp);  in irdma_create_qp()
1010 			irdma_destroy_qp(&iwqp->ibqp, udata);  in irdma_create_qp()
1015 	init_completion(&iwqp->free_qp);  in irdma_create_qp()
1019 	irdma_free_qp_rsrc(iwqp);  in irdma_create_qp()
1023 static int irdma_get_ib_acc_flags(struct irdma_qp *iwqp)  in irdma_get_ib_acc_flags()  argument
1027 	if (rdma_protocol_roce(iwqp->ibqp.device, 1)) {  in irdma_get_ib_acc_flags()
1028 		if (iwqp->roce_info.wr_rdresp_en) {  in irdma_get_ib_acc_flags()
1032 		if (iwqp->roce_info.rd_en)  in irdma_get_ib_acc_flags()
1034 		if (iwqp->roce_info.bind_en)  in irdma_get_ib_acc_flags()
1037 		if (iwqp->iwarp_info.wr_rdresp_en) {  in irdma_get_ib_acc_flags()
1041 		if (iwqp->iwarp_info.rd_en)  in irdma_get_ib_acc_flags()
1043 		if (iwqp->iwarp_info.bind_en)  in irdma_get_ib_acc_flags()
1059 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_query_qp()  local
1060 	struct irdma_sc_qp *qp = &iwqp->sc_qp;  in irdma_query_qp()
1065 	attr->qp_state = iwqp->ibqp_state;  in irdma_query_qp()
1066 	attr->cur_qp_state = iwqp->ibqp_state;  in irdma_query_qp()
1067 	attr->cap.max_send_wr = iwqp->max_send_wr;  in irdma_query_qp()
1068 	attr->cap.max_recv_wr = iwqp->max_recv_wr;  in irdma_query_qp()
1072 	attr->qp_access_flags = irdma_get_ib_acc_flags(iwqp);  in irdma_query_qp()
1075 		attr->path_mtu = ib_mtu_int_to_enum(iwqp->udp_info.snd_mss);  in irdma_query_qp()
1076 		attr->qkey = iwqp->roce_info.qkey;  in irdma_query_qp()
1077 		attr->rq_psn = iwqp->udp_info.epsn;  in irdma_query_qp()
1078 		attr->sq_psn = iwqp->udp_info.psn_nxt;  in irdma_query_qp()
1079 		attr->dest_qp_num = iwqp->roce_info.dest_qp;  in irdma_query_qp()
1080 		attr->pkey_index = iwqp->roce_info.p_key;  in irdma_query_qp()
1081 		attr->retry_cnt = iwqp->udp_info.rexmit_thresh;  in irdma_query_qp()
1082 		attr->rnr_retry = iwqp->udp_info.rnr_nak_thresh;  in irdma_query_qp()
1083 		attr->max_rd_atomic = iwqp->roce_info.ord_size;  in irdma_query_qp()
1084 		attr->max_dest_rd_atomic = iwqp->roce_info.ird_size;  in irdma_query_qp()
1087 	init_attr->event_handler = iwqp->ibqp.event_handler;  in irdma_query_qp()
1088 	init_attr->qp_context = iwqp->ibqp.qp_context;  in irdma_query_qp()
1089 	init_attr->send_cq = iwqp->ibqp.send_cq;  in irdma_query_qp()
1090 	init_attr->recv_cq = iwqp->ibqp.recv_cq;  in irdma_query_qp()
1124 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_modify_qp_roce()  local
1125 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp_roce()
1137 	ctx_info = &iwqp->ctx_info;  in irdma_modify_qp_roce()
1138 	roce_info = &iwqp->roce_info;  in irdma_modify_qp_roce()
1139 	udp_info = &iwqp->udp_info;  in irdma_modify_qp_roce()
1179 		struct irdma_av *av = &iwqp->roce_ah.av;  in irdma_modify_qp_roce()
1184 		memset(&iwqp->roce_ah, 0, sizeof(iwqp->roce_ah));  in irdma_modify_qp_roce()
1193 			irdma_qp_rem_qos(&iwqp->sc_qp);  in irdma_modify_qp_roce()
1194 			dev->ws_remove(iwqp->sc_qp.vsi, ctx_info->user_pri);  in irdma_modify_qp_roce()
1196 			iwqp->sc_qp.user_pri = ctx_info->user_pri;  in irdma_modify_qp_roce()
1197 			if (dev->ws_add(iwqp->sc_qp.vsi, ctx_info->user_pri))  in irdma_modify_qp_roce()
1199 			irdma_qp_add_qos(&iwqp->sc_qp);  in irdma_modify_qp_roce()
1292 	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));  in irdma_modify_qp_roce()
1297 		  iwqp->ibqp_state, iwqp->iwarp_state, attr_mask);  in irdma_modify_qp_roce()
1299 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1301 		if (!ib_modify_qp_is_ok(iwqp->ibqp_state, attr->qp_state,  in irdma_modify_qp_roce()
1302 					iwqp->ibqp.qp_type, attr_mask)) {  in irdma_modify_qp_roce()
1304 				   iwqp->ibqp.qp_num, iwqp->ibqp_state,  in irdma_modify_qp_roce()
1309 		info.curr_iwarp_state = iwqp->iwarp_state;  in irdma_modify_qp_roce()
1313 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp_roce()
1318 			if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {  in irdma_modify_qp_roce()
1324 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp_roce()
1334 			if (iwqp->ibqp_state < IB_QPS_RTR ||  in irdma_modify_qp_roce()
1335 			    iwqp->ibqp_state == IB_QPS_ERR) {  in irdma_modify_qp_roce()
1346 			    iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp_roce()
1348 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1349 				irdma_alloc_push_page(iwqp);  in irdma_modify_qp_roce()
1350 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1354 			if (iwqp->iwarp_state == IRDMA_QP_STATE_SQD)  in irdma_modify_qp_roce()
1357 			if (iwqp->iwarp_state != IRDMA_QP_STATE_RTS) {  in irdma_modify_qp_roce()
1368 			if (iwqp->iwarp_state == IRDMA_QP_STATE_RTS) {  in irdma_modify_qp_roce()
1369 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1371 				irdma_hw_modify_qp(iwdev, iwqp, &info, true);  in irdma_modify_qp_roce()
1372 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1375 			if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {  in irdma_modify_qp_roce()
1376 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1382 					irdma_flush_wqes(iwqp,  in irdma_modify_qp_roce()
1398 		iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1401 	ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp_roce()
1402 	ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp_roce()
1403 	irdma_sc_qp_setctx_roce(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_modify_qp_roce()
1404 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1409 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp_roce()
1411 			spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1412 			if (iwqp->iwarp_state == info.curr_iwarp_state) {  in irdma_modify_qp_roce()
1413 				iwqp->iwarp_state = info.next_iwarp_state;  in irdma_modify_qp_roce()
1414 				iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1416 			if (iwqp->ibqp_state > IB_QPS_RTS &&  in irdma_modify_qp_roce()
1417 			    !iwqp->flush_issued) {  in irdma_modify_qp_roce()
1418 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1419 				irdma_flush_wqes(iwqp, IRDMA_FLUSH_SQ |  in irdma_modify_qp_roce()
1422 				iwqp->flush_issued = 1;  in irdma_modify_qp_roce()
1424 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1427 			iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp_roce()
1434 			if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp_roce()
1435 			    !iwqp->push_wqe_mmap_entry &&  in irdma_modify_qp_roce()
1436 			    !irdma_setup_push_mmap_entries(ucontext, iwqp,  in irdma_modify_qp_roce()
1439 				uresp.push_offset = iwqp->sc_qp.push_offset;  in irdma_modify_qp_roce()
1444 				irdma_remove_push_mmap_entries(iwqp);  in irdma_modify_qp_roce()
1454 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp_roce()
1469 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_modify_qp()  local
1470 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_modify_qp()
1486 	ctx_info = &iwqp->ctx_info;  in irdma_modify_qp()
1487 	offload_info = &iwqp->iwarp_info;  in irdma_modify_qp()
1488 	tcp_info = &iwqp->tcp_info;  in irdma_modify_qp()
1489 	wait_event(iwqp->mod_qp_waitq, !atomic_read(&iwqp->hw_mod_qp_pend));  in irdma_modify_qp()
1493 		  iwqp->ibqp_state, iwqp->iwarp_state, iwqp->last_aeq,  in irdma_modify_qp()
1494 		  iwqp->hw_tcp_state, iwqp->hw_iwarp_state, attr_mask);  in irdma_modify_qp()
1496 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1498 		info.curr_iwarp_state = iwqp->iwarp_state;  in irdma_modify_qp()
1502 			if (iwqp->iwarp_state > IRDMA_QP_STATE_IDLE) {  in irdma_modify_qp()
1507 			if (iwqp->iwarp_state == IRDMA_QP_STATE_INVALID) {  in irdma_modify_qp()
1512 			    iwqp->sc_qp.push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp()
1514 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1515 				irdma_alloc_push_page(iwqp);  in irdma_modify_qp()
1516 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1520 			if (iwqp->iwarp_state > IRDMA_QP_STATE_RTS ||  in irdma_modify_qp()
1521 			    !iwqp->cm_id) {  in irdma_modify_qp()
1527 			iwqp->hw_tcp_state = IRDMA_TCP_STATE_ESTABLISHED;  in irdma_modify_qp()
1528 			iwqp->hte_added = 1;  in irdma_modify_qp()
1536 			if (iwqp->hw_iwarp_state > IRDMA_QP_STATE_RTS) {  in irdma_modify_qp()
1541 			if (iwqp->iwarp_state == IRDMA_QP_STATE_CLOSING ||  in irdma_modify_qp()
1542 			    iwqp->iwarp_state < IRDMA_QP_STATE_RTS) {  in irdma_modify_qp()
1547 			if (iwqp->iwarp_state > IRDMA_QP_STATE_CLOSING) {  in irdma_modify_qp()
1556 			if (iwqp->iwarp_state >= IRDMA_QP_STATE_TERMINATE) {  in irdma_modify_qp()
1566 			if (iwqp->iwarp_state == IRDMA_QP_STATE_ERROR) {  in irdma_modify_qp()
1567 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1573 					irdma_flush_wqes(iwqp,  in irdma_modify_qp()
1581 			if (iwqp->sc_qp.term_flags) {  in irdma_modify_qp()
1582 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1583 				irdma_terminate_del_timer(&iwqp->sc_qp);  in irdma_modify_qp()
1584 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1587 			if (iwqp->hw_tcp_state > IRDMA_TCP_STATE_CLOSED &&  in irdma_modify_qp()
1589 			    iwqp->hw_tcp_state != IRDMA_TCP_STATE_TIME_WAIT)  in irdma_modify_qp()
1602 		iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp()
1615 		ctx_info->send_cq_num = iwqp->iwscq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp()
1616 		ctx_info->rcv_cq_num = iwqp->iwrcq->sc_cq.cq_uk.cq_id;  in irdma_modify_qp()
1617 		irdma_sc_qp_setctx(&iwqp->sc_qp, iwqp->host_ctx.va, ctx_info);  in irdma_modify_qp()
1619 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1624 			if (irdma_hw_modify_qp(iwdev, iwqp, &info, true))  in irdma_modify_qp()
1628 		spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1629 		if (iwqp->iwarp_state == info.curr_iwarp_state) {  in irdma_modify_qp()
1630 			iwqp->iwarp_state = info.next_iwarp_state;  in irdma_modify_qp()
1631 			iwqp->ibqp_state = attr->qp_state;  in irdma_modify_qp()
1633 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1636 	if (issue_modify_qp && iwqp->ibqp_state > IB_QPS_RTS) {  in irdma_modify_qp()
1638 			if (iwqp->hw_tcp_state) {  in irdma_modify_qp()
1639 				spin_lock_irqsave(&iwqp->lock, flags);  in irdma_modify_qp()
1640 				iwqp->hw_tcp_state = IRDMA_TCP_STATE_CLOSED;  in irdma_modify_qp()
1641 				iwqp->last_aeq = IRDMA_AE_RESET_SENT;  in irdma_modify_qp()
1642 				spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
1644 			irdma_cm_disconn(iwqp);  in irdma_modify_qp()
1650 			if (iwqp->cm_node) {  in irdma_modify_qp()
1651 				refcount_inc(&iwqp->cm_node->refcnt);  in irdma_modify_qp()
1653 				close_timer_started = atomic_inc_return(&iwqp->close_timer_started);  in irdma_modify_qp()
1654 				if (iwqp->cm_id && close_timer_started == 1)  in irdma_modify_qp()
1655 					irdma_schedule_cm_timer(iwqp->cm_node,  in irdma_modify_qp()
1656 						(struct irdma_puda_buf *)iwqp,  in irdma_modify_qp()
1659 				irdma_rem_ref_cm_node(iwqp->cm_node);  in irdma_modify_qp()
1671 		if (iwqp->sc_qp.push_idx != IRDMA_INVALID_PUSH_PAGE_INDEX &&  in irdma_modify_qp()
1672 		    !iwqp->push_wqe_mmap_entry &&  in irdma_modify_qp()
1673 		    !irdma_setup_push_mmap_entries(ucontext, iwqp,  in irdma_modify_qp()
1676 			uresp.push_offset = iwqp->sc_qp.push_offset;  in irdma_modify_qp()
1682 			irdma_remove_push_mmap_entries(iwqp);  in irdma_modify_qp()
1691 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_modify_qp()
3063 	struct irdma_qp *iwqp;  in irdma_post_send()  local
3072 	iwqp = to_iwqp(ibqp);  in irdma_post_send()
3073 	ukqp = &iwqp->sc_qp.qp_uk;  in irdma_post_send()
3074 	dev = &iwqp->iwdev->rf->sc_dev;  in irdma_post_send()
3076 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_post_send()
3081 		if ((ib_wr->send_flags & IB_SEND_SIGNALED) || iwqp->sig_all)  in irdma_post_send()
3115 				if (iwqp->ibqp.qp_type == IB_QPT_UD ||  in irdma_post_send()
3116 				    iwqp->ibqp.qp_type == IB_QPT_GSI) {  in irdma_post_send()
3126 				if (iwqp->ibqp.qp_type == IB_QPT_UD ||  in irdma_post_send()
3127 				    iwqp->ibqp.qp_type == IB_QPT_GSI) {  in irdma_post_send()
3209 			err = irdma_sc_mr_fast_register(&iwqp->sc_qp, &stag_info,  in irdma_post_send()
3215 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_send()
3226 	if (!iwqp->flush_issued) {  in irdma_post_send()
3227 		if (iwqp->hw_iwarp_state <= IRDMA_QP_STATE_RTS)  in irdma_post_send()
3229 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_send()
3231 		spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_send()
3232 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_send()
3251 	struct irdma_qp *iwqp;  in irdma_post_recv()  local
3257 	iwqp = to_iwqp(ibqp);  in irdma_post_recv()
3258 	ukqp = &iwqp->sc_qp.qp_uk;  in irdma_post_recv()
3260 	spin_lock_irqsave(&iwqp->lock, flags);  in irdma_post_recv()
3267 			ibdev_dbg(&iwqp->iwdev->ibdev,  in irdma_post_recv()
3276 	spin_unlock_irqrestore(&iwqp->lock, flags);  in irdma_post_recv()
3277 	if (iwqp->flush_issued)  in irdma_post_recv()
3278 		mod_delayed_work(iwqp->iwdev->cleanup_wq, &iwqp->dwork_flush,  in irdma_post_recv()
3324 	struct irdma_qp *iwqp;  in irdma_process_cqe()  local
3332 	iwqp = qp->qp_uk.back_qp;  in irdma_process_cqe()
3397 		ibdev_err(&iwqp->iwdev->ibdev,  in irdma_process_cqe()
3892 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_attach_mcast()  local
3893 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_attach_mcast()
3971 			iwqp->sc_qp.vsi->qos[iwqp->sc_qp.user_pri].qs_handle;  in irdma_attach_mcast()
3984 	mcg_info.qp_id = iwqp->ibqp.qp_num;  in irdma_attach_mcast()
4032 	struct irdma_qp *iwqp = to_iwqp(ibqp);  in irdma_detach_mcast()  local
4033 	struct irdma_device *iwdev = iwqp->iwdev;  in irdma_detach_mcast()
4058 	mcg_info.qp_id = iwqp->ibqp.qp_num;  in irdma_detach_mcast()