214 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_bind_mw()  local
221 	if (nesqp->ibqp_state > IB_QPS_RTS)  in nes_bind_mw()
224 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_bind_mw()
226 	head = nesqp->hwqp.sq_head;  in nes_bind_mw()
227 	qsize = nesqp->hwqp.sq_tail;  in nes_bind_mw()
230 	if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {  in nes_bind_mw()
231 		spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_bind_mw()
235 	wqe = &nesqp->hwqp.sq_vbase[head];  in nes_bind_mw()
237 	nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_bind_mw()
267 	nesqp->hwqp.sq_head = head;  in nes_bind_mw()
271 			(1 << 24) | 0x00800000 | nesqp->hwqp.qp_id);  in nes_bind_mw()
273 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_bind_mw()
727 	struct nes_qp *nesqp;  in nes_mmap()  local
740 		nesqp = nes_ucontext->mmap_nesqp[index];  in nes_mmap()
741 		if (nesqp == NULL) {  in nes_mmap()
746 				virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT,  in nes_mmap()
752 		vma->vm_private_data = nesqp;  in nes_mmap()
914 static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl,  in nes_setup_virt_qp()  argument
938 …nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1])…  in nes_setup_virt_qp()
945 	nesqp->hwqp.sq_vbase = kmap(nespbl->page);  in nes_setup_virt_qp()
946 	nesqp->page = nespbl->page;  in nes_setup_virt_qp()
947 	if (!nesqp->hwqp.sq_vbase) {  in nes_setup_virt_qp()
956 …nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1])…  in nes_setup_virt_qp()
961 		  nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase,  in nes_setup_virt_qp()
962 		  nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase);  in nes_setup_virt_qp()
968 		kunmap(nesqp->page);  in nes_setup_virt_qp()
975 	nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase);  in nes_setup_virt_qp()
976 	pblbuffer = nesqp->pbl_vbase;  in nes_setup_virt_qp()
977 	if (!nesqp->pbl_vbase) {  in nes_setup_virt_qp()
985 		kunmap(nesqp->page);  in nes_setup_virt_qp()
988 	memset(nesqp->pbl_vbase, 0, 256);  in nes_setup_virt_qp()
1003 	nesqp->qp_mem_size =  in nes_setup_virt_qp()
1006 	nesqp->qp_mem_size += PAGE_SIZE - 1;  in nes_setup_virt_qp()
1007 	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);  in nes_setup_virt_qp()
1009 	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_setup_virt_qp()
1010 			&nesqp->hwqp.q2_pbase);  in nes_setup_virt_qp()
1013 		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase);  in nes_setup_virt_qp()
1014 		nesqp->pbl_vbase = NULL;  in nes_setup_virt_qp()
1018 		kunmap(nesqp->page);  in nes_setup_virt_qp()
1021 	nesqp->sq_kmapped = 1;  in nes_setup_virt_qp()
1022 	nesqp->hwqp.q2_vbase = mem;  in nes_setup_virt_qp()
1024 	memset(nesqp->hwqp.q2_vbase, 0, 256);  in nes_setup_virt_qp()
1025 	nesqp->nesqp_context = mem;  in nes_setup_virt_qp()
1026 	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));  in nes_setup_virt_qp()
1027 	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;  in nes_setup_virt_qp()
1036 static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic,  in nes_setup_mmap_qp()  argument
1042 	nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) +  in nes_setup_mmap_qp()
1047 	nesqp->qp_mem_size += PAGE_SIZE - 1;  in nes_setup_mmap_qp()
1048 	nesqp->qp_mem_size &= ~(PAGE_SIZE - 1);  in nes_setup_mmap_qp()
1050 	mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_setup_mmap_qp()
1051 			&nesqp->hwqp.sq_pbase);  in nes_setup_mmap_qp()
1056 			mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size);  in nes_setup_mmap_qp()
1058 	memset(mem, 0, nesqp->qp_mem_size);  in nes_setup_mmap_qp()
1060 	nesqp->hwqp.sq_vbase = mem;  in nes_setup_mmap_qp()
1063 	nesqp->hwqp.rq_vbase = mem;  in nes_setup_mmap_qp()
1064 	nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase +  in nes_setup_mmap_qp()
1068 	nesqp->hwqp.q2_vbase = mem;  in nes_setup_mmap_qp()
1069 	nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase +  in nes_setup_mmap_qp()
1072 	memset(nesqp->hwqp.q2_vbase, 0, 256);  in nes_setup_mmap_qp()
1074 	nesqp->nesqp_context = mem;  in nes_setup_mmap_qp()
1075 	nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256;  in nes_setup_mmap_qp()
1076 	memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context));  in nes_setup_mmap_qp()
1085 		struct nes_qp *nesqp, int virt_wqs)  in nes_free_qp_mem()  argument
1090 		pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size,  in nes_free_qp_mem()
1091 				nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase);  in nes_free_qp_mem()
1096 …pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase…  in nes_free_qp_mem()
1097 		pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase );  in nes_free_qp_mem()
1098 		nesqp->pbl_vbase = NULL;  in nes_free_qp_mem()
1099 		if (nesqp->sq_kmapped) {  in nes_free_qp_mem()
1100 			nesqp->sq_kmapped = 0;  in nes_free_qp_mem()
1101 			kunmap(nesqp->page);  in nes_free_qp_mem()
1119 	struct nes_qp *nesqp;  in nes_create_qp()  local
1176 			mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL);  in nes_create_qp()
1186 			nesqp = (struct nes_qp *)(unsigned long)u64nesqp;  in nes_create_qp()
1189 			nesqp->allocated_buffer = mem;  in nes_create_qp()
1194 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1202 					nesqp->user_mode = 1;  in nes_create_qp()
1219 							kfree(nesqp->allocated_buffer);  in nes_create_qp()
1225 					nesqp->mmap_sq_db_index =  in nes_create_qp()
1230 					if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) {  in nes_create_qp()
1239 						kfree(nesqp->allocated_buffer);  in nes_create_qp()
1242 					set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);  in nes_create_qp()
1243 					nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp;  in nes_create_qp()
1244 					nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1;  in nes_create_qp()
1247 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1251 			err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) :  in nes_create_qp()
1252 					nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size);  in nes_create_qp()
1257 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1261 			nesqp->hwqp.sq_size = sq_size;  in nes_create_qp()
1262 			nesqp->hwqp.sq_encoded_size = sq_encoded_size;  in nes_create_qp()
1263 			nesqp->hwqp.sq_head = 1;  in nes_create_qp()
1264 			nesqp->hwqp.rq_size = rq_size;  in nes_create_qp()
1265 			nesqp->hwqp.rq_encoded_size = rq_encoded_size;  in nes_create_qp()
1269 			nesqp->hwqp.qp_id = qp_num;  in nes_create_qp()
1270 			nesqp->ibqp.qp_num = nesqp->hwqp.qp_id;  in nes_create_qp()
1271 			nesqp->nespd = nespd;  in nes_create_qp()
1274 			nesqp->nesscq = nescq;  in nes_create_qp()
1276 			nesqp->nesrcq = nescq;  in nes_create_qp()
1278 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) <<  in nes_create_qp()
1280 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size <<  in nes_create_qp()
1282 			nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size <<  in nes_create_qp()
1285 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN);  in nes_create_qp()
1286 				nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN);  in nes_create_qp()
1288 			nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number +  in nes_create_qp()
1289 					((u32)nesqp->nesrcq->hw_cq.cq_number << 16));  in nes_create_qp()
1290 			u64temp = (u64)nesqp->hwqp.sq_pbase;  in nes_create_qp()
1291 			nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1292 			nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1296 				u64temp = (u64)nesqp->hwqp.sq_pbase;  in nes_create_qp()
1297 				nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1298 				nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1299 				u64temp = (u64)nesqp->hwqp.rq_pbase;  in nes_create_qp()
1300 				nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1301 				nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1303 				u64temp = (u64)nesqp->pbl_pbase;  in nes_create_qp()
1304 				nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1305 				nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1312 			nesqp->nesqp_context->misc2 |= cpu_to_le32(  in nes_create_qp()
1322 			nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16);  in nes_create_qp()
1323 			u64temp = (u64)nesqp->hwqp.q2_pbase;  in nes_create_qp()
1324 			nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp);  in nes_create_qp()
1325 			nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32));  in nes_create_qp()
1326 			nesqp->nesqp_context->aeq_token_low =  cpu_to_le32((u32)((unsigned long)(nesqp)));  in nes_create_qp()
1327 …	nesqp->nesqp_context->aeq_token_high =  cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp))));  in nes_create_qp()
1328 			nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM |  in nes_create_qp()
1334 				nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC);  in nes_create_qp()
1343 				nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1344 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1360 			set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);  in nes_create_qp()
1362 			u64temp = (u64)nesqp->nesqp_context_pbase;  in nes_create_qp()
1370 					nesqp->hwqp.qp_id);  in nes_create_qp()
1376 					nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail,  in nes_create_qp()
1381 				nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1382 				kfree(nesqp->allocated_buffer);  in nes_create_qp()
1393 				uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index;  in nes_create_qp()
1396 				uresp.qp_id = nesqp->hwqp.qp_id;  in nes_create_qp()
1400 					nes_free_qp_mem(nesdev, nesqp,virt_wqs);  in nes_create_qp()
1401 					kfree(nesqp->allocated_buffer);  in nes_create_qp()
1407 					nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp));  in nes_create_qp()
1408 			spin_lock_init(&nesqp->lock);  in nes_create_qp()
1409 			nes_add_ref(&nesqp->ibqp);  in nes_create_qp()
1416 	nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR);  in nes_create_qp()
1419 	nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp;  in nes_create_qp()
1423 	return &nesqp->ibqp;  in nes_create_qp()
1430 static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq)  in nes_clean_cq()  argument
1447 		if (u64temp == (u64)(unsigned long)nesqp) {  in nes_clean_cq()
1466 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_destroy_qp()  local
1474 	nesqp->destroyed = 1;  in nes_destroy_qp()
1477 	if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) {  in nes_destroy_qp()
1480 		nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL);  in nes_destroy_qp()
1483 	if (((nesqp->ibqp_state == IB_QPS_INIT) ||  in nes_destroy_qp()
1484 			(nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) {  in nes_destroy_qp()
1485 		cm_id = nesqp->cm_id;  in nes_destroy_qp()
1495 				nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount));  in nes_destroy_qp()
1503 	if (nesqp->user_mode) {  in nes_destroy_qp()
1506 			clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs);  in nes_destroy_qp()
1507 			nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL;  in nes_destroy_qp()
1508 			if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) {  in nes_destroy_qp()
1509 				nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index;  in nes_destroy_qp()
1512 		if (nesqp->pbl_pbase && nesqp->sq_kmapped) {  in nes_destroy_qp()
1513 			nesqp->sq_kmapped = 0;  in nes_destroy_qp()
1514 			kunmap(nesqp->page);  in nes_destroy_qp()
1518 		if (nesqp->nesscq)  in nes_destroy_qp()
1519 			nes_clean_cq(nesqp, nesqp->nesscq);  in nes_destroy_qp()
1521 		if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq))  in nes_destroy_qp()
1522 			nes_clean_cq(nesqp, nesqp->nesrcq);  in nes_destroy_qp()
1525 	nes_rem_ref(&nesqp->ibqp);  in nes_destroy_qp()
2817 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_query_qp()  local
2822 	attr->cap.max_send_wr = nesqp->hwqp.sq_size;  in nes_query_qp()
2823 	attr->cap.max_recv_wr = nesqp->hwqp.rq_size;  in nes_query_qp()
2830 	init_attr->event_handler = nesqp->ibqp.event_handler;  in nes_query_qp()
2831 	init_attr->qp_context = nesqp->ibqp.qp_context;  in nes_query_qp()
2832 	init_attr->send_cq = nesqp->ibqp.send_cq;  in nes_query_qp()
2833 	init_attr->recv_cq = nesqp->ibqp.recv_cq;  in nes_query_qp()
2834 	init_attr->srq = nesqp->ibqp.srq = nesqp->ibqp.srq;  in nes_query_qp()
2844 int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp,  in nes_hw_modify_qp()  argument
2855 			nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_hw_modify_qp()
2874 	set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id);  in nes_hw_modify_qp()
2875 …set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_…  in nes_hw_modify_qp()
2895 				nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code);  in nes_hw_modify_qp()
2900 					nesqp->hwqp.qp_id, cqp_request->major_code,  in nes_hw_modify_qp()
2924 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_modify_qp()  local
2939 			nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state,  in nes_modify_qp()
2940 			nesqp->iwarp_state, atomic_read(&nesqp->refcount));  in nes_modify_qp()
2942 	spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
2946 			nesqp->hwqp.qp_id, nesqp->hw_iwarp_state,  in nes_modify_qp()
2947 			nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask);  in nes_modify_qp()
2953 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2954 				if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) {  in nes_modify_qp()
2955 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2963 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2964 				if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) {  in nes_modify_qp()
2965 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2973 						nesqp->hwqp.qp_id);  in nes_modify_qp()
2974 				if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) {  in nes_modify_qp()
2975 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2978 				if (nesqp->cm_id == NULL) {  in nes_modify_qp()
2980 							nesqp->hwqp.qp_id );  in nes_modify_qp()
2981 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
2985 				if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS)  in nes_modify_qp()
2989 				nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED;  in nes_modify_qp()
2990 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS;  in nes_modify_qp()
2991 				nesqp->hte_added = 1;  in nes_modify_qp()
2996 						nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail);  in nes_modify_qp()
2997 				if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {  in nes_modify_qp()
2998 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3001 					if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) {  in nes_modify_qp()
3004 								nesqp->hwqp.qp_id);  in nes_modify_qp()
3005 						spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3008 					if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) {  in nes_modify_qp()
3011 								nesqp->hwqp.qp_id);  in nes_modify_qp()
3014 					switch (nesqp->hw_iwarp_state) {  in nes_modify_qp()
3025 							nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING;  in nes_modify_qp()
3032 						nesqp->hwqp.qp_id);  in nes_modify_qp()
3033 				if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) {  in nes_modify_qp()
3034 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3039 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE;  in nes_modify_qp()
3044 				if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) {  in nes_modify_qp()
3045 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3049 						nesqp->hwqp.qp_id);  in nes_modify_qp()
3050 				if (nesqp->term_flags)  in nes_modify_qp()
3051 					del_timer(&nesqp->terminate_timer);  in nes_modify_qp()
3055 					if (nesqp->hte_added) {  in nes_modify_qp()
3058 						nesqp->hte_added = 0;  in nes_modify_qp()
3060 				if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) &&  in nes_modify_qp()
3062 						(nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) {  in nes_modify_qp()
3066 							nesqp->hwqp.qp_id, nesqp->hw_tcp_state);  in nes_modify_qp()
3070 				nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR;  in nes_modify_qp()
3073 				spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3078 		nesqp->ibqp_state = attr->qp_state;  in nes_modify_qp()
3079 		if (((nesqp->iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) ==  in nes_modify_qp()
3083 			nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK;  in nes_modify_qp()
3085 					nesqp->iwarp_state);  in nes_modify_qp()
3087 			nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK;  in nes_modify_qp()
3089 					nesqp->iwarp_state);  in nes_modify_qp()
3095 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |  in nes_modify_qp()
3100 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN);  in nes_modify_qp()
3104 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN);  in nes_modify_qp()
3108 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN);  in nes_modify_qp()
3112 		if (nesqp->user_mode) {  in nes_modify_qp()
3113 			nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN |  in nes_modify_qp()
3119 	original_last_aeq = nesqp->last_aeq;  in nes_modify_qp()
3120 	spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3129 		ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1);  in nes_modify_qp()
3133 					next_iwarp_state, nesqp->hwqp.qp_id);  in nes_modify_qp()
3137 	if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) {  in nes_modify_qp()
3140 				nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3141 				original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3146 				if (nesqp->cm_id && nesqp->hw_tcp_state != 0) {  in nes_modify_qp()
3149 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3150 							original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3152 					spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
3153 					nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED;  in nes_modify_qp()
3154 					nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT;  in nes_modify_qp()
3155 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3156 					nes_cm_disconn(nesqp);  in nes_modify_qp()
3159 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_modify_qp()
3162 				spin_lock_irqsave(&nesqp->lock, qplockflags);  in nes_modify_qp()
3163 				if (nesqp->cm_id) {  in nes_modify_qp()
3165 					if (atomic_inc_return(&nesqp->close_timer_started) == 1) {  in nes_modify_qp()
3166 						nesqp->cm_id->add_ref(nesqp->cm_id);  in nes_modify_qp()
3170 								nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3171 								original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3172 						schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0);  in nes_modify_qp()
3174 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3176 					spin_unlock_irqrestore(&nesqp->lock, qplockflags);  in nes_modify_qp()
3180 							nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3181 							original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3187 					nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3188 					original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3193 				nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount),  in nes_modify_qp()
3194 				original_last_aeq, nesqp->last_aeq);  in nes_modify_qp()
3200 			nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount));  in nes_modify_qp()
3271 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_post_send()  local
3274 	u32 qsize = nesqp->hwqp.sq_size;  in nes_post_send()
3280 	if (nesqp->ibqp_state > IB_QPS_RTS) {  in nes_post_send()
3285 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_post_send()
3287 	head = nesqp->hwqp.sq_head;  in nes_post_send()
3291 		if (nesqp->term_flags) {  in nes_post_send()
3297 		if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) {  in nes_post_send()
3302 		wqe = &nesqp->hwqp.sq_vbase[head];  in nes_post_send()
3305 		nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_post_send()
3508 		if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all)  in nes_post_send()
3521 	nesqp->hwqp.sq_head = head;  in nes_post_send()
3527 				(counter << 24) | 0x00800000 | nesqp->hwqp.qp_id);  in nes_post_send()
3530 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_post_send()
3549 	struct nes_qp *nesqp = to_nesqp(ibqp);  in nes_post_recv()  local
3553 	u32 qsize = nesqp->hwqp.rq_size;  in nes_post_recv()
3559 	if (nesqp->ibqp_state > IB_QPS_RTS) {  in nes_post_recv()
3564 	spin_lock_irqsave(&nesqp->lock, flags);  in nes_post_recv()
3566 	head = nesqp->hwqp.rq_head;  in nes_post_recv()
3570 		if (nesqp->term_flags) {  in nes_post_recv()
3580 		if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) {  in nes_post_recv()
3586 		wqe = &nesqp->hwqp.rq_vbase[head];  in nes_post_recv()
3590 		nes_fill_init_qp_wqe(wqe, nesqp, head);  in nes_post_recv()
3615 	nesqp->hwqp.rq_head = head;  in nes_post_recv()
3620 		nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id);  in nes_post_recv()
3623 	spin_unlock_irqrestore(&nesqp->lock, flags);  in nes_post_recv()
3643 	struct nes_qp *nesqp;  in nes_poll_cq()  local
3681 			nesqp = (struct nes_qp *)(unsigned long)u64temp;  in nes_poll_cq()
3698 			entry->qp = &nesqp->ibqp;  in nes_poll_cq()
3699 			entry->src_qp = nesqp->hwqp.qp_id;  in nes_poll_cq()
3702 				if (nesqp->skip_lsmm) {  in nes_poll_cq()
3703 					nesqp->skip_lsmm = 0;  in nes_poll_cq()
3704 					nesqp->hwqp.sq_tail++;  in nes_poll_cq()
3708 				wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3710 						((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3712 				entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3715 				switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3724 						entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index].  in nes_poll_cq()
3742 				nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1);  in nes_poll_cq()
3743 				if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) {  in nes_poll_cq()
3745 					wq_tail = nesqp->hwqp.sq_tail;  in nes_poll_cq()
3750 …wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_…  in nes_poll_cq()
3751 …((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_ID…  in nes_poll_cq()
3754 				nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1);  in nes_poll_cq()
3755 				if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) {  in nes_poll_cq()
3757 					wq_tail = nesqp->hwqp.rq_tail;  in nes_poll_cq()