824 		       struct lpfc_io_buf *lpfc_ncmd,  in lpfc_nvme_adj_fcp_sgls()  argument
838 	wqe = &lpfc_ncmd->cur_iocbq.wqe;  in lpfc_nvme_adj_fcp_sgls()
847 	sgl = lpfc_ncmd->dma_sgl;  in lpfc_nvme_adj_fcp_sgls()
934 	struct lpfc_io_buf *lpfc_ncmd = pwqeIn->io_buf;  in lpfc_nvme_io_cmd_cmpl()  local
954 	if (!lpfc_ncmd) {  in lpfc_nvme_io_cmd_cmpl()
962 	spin_lock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_io_cmd_cmpl()
964 	if (!lpfc_ncmd->nvmeCmd) {  in lpfc_nvme_io_cmd_cmpl()
965 		spin_unlock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_io_cmd_cmpl()
969 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd);  in lpfc_nvme_io_cmd_cmpl()
972 		lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_cmpl()
975 	nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_io_cmd_cmpl()
978 	idx = lpfc_ncmd->cur_iocbq.hba_wqidx;  in lpfc_nvme_io_cmd_cmpl()
991 			 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_cmpl()
997 	ndlp = lpfc_ncmd->ndlp;  in lpfc_nvme_io_cmd_cmpl()
1038 		lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_nvme_io_cmd_cmpl()
1039 		lpfc_ncmd->result = 0;  in lpfc_nvme_io_cmd_cmpl()
1043 		lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);  in lpfc_nvme_io_cmd_cmpl()
1044 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);  in lpfc_nvme_io_cmd_cmpl()
1057 		switch (lpfc_ncmd->status) {  in lpfc_nvme_io_cmd_cmpl()
1074 				lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_nvme_io_cmd_cmpl()
1075 				lpfc_ncmd->result = 0;  in lpfc_nvme_io_cmd_cmpl()
1081 					lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_cmpl()
1093 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_cmpl()
1094 					 lpfc_ncmd->status, lpfc_ncmd->result,  in lpfc_nvme_io_cmd_cmpl()
1102 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)  in lpfc_nvme_io_cmd_cmpl()
1108 					 lpfc_ncmd, nCmd,  in lpfc_nvme_io_cmd_cmpl()
1109 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_cmpl()
1118 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_cmpl()
1119 					 lpfc_ncmd->status, lpfc_ncmd->result,  in lpfc_nvme_io_cmd_cmpl()
1131 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;  in lpfc_nvme_io_cmd_cmpl()
1133 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;  in lpfc_nvme_io_cmd_cmpl()
1140 	if (lpfc_ncmd->ts_cmd_start) {  in lpfc_nvme_io_cmd_cmpl()
1141 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;  in lpfc_nvme_io_cmd_cmpl()
1142 		lpfc_ncmd->ts_data_io = ktime_get_ns();  in lpfc_nvme_io_cmd_cmpl()
1143 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_io;  in lpfc_nvme_io_cmd_cmpl()
1144 		lpfc_io_ktime(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_cmpl()
1149 		if (lpfc_ncmd->cpu != cpu)  in lpfc_nvme_io_cmd_cmpl()
1154 					 cpu, lpfc_ncmd->cpu);  in lpfc_nvme_io_cmd_cmpl()
1162 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {  in lpfc_nvme_io_cmd_cmpl()
1165 		lpfc_ncmd->nvmeCmd = NULL;  in lpfc_nvme_io_cmd_cmpl()
1168 	spin_unlock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_io_cmd_cmpl()
1175 		lat = ktime_get_ns() - lpfc_ncmd->rx_cmd_start;  in lpfc_nvme_io_cmd_cmpl()
1183 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_cmpl()
1204 		      struct lpfc_io_buf *lpfc_ncmd,  in lpfc_nvme_prep_io_cmd()  argument
1209 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_prep_io_cmd()
1211 	struct lpfc_iocbq *pwqeq = &lpfc_ncmd->cur_iocbq;  in lpfc_nvme_prep_io_cmd()
1232 				req_len = lpfc_ncmd->nvmeCmd->payload_length;  in lpfc_nvme_prep_io_cmd()
1302 	if (unlikely(lpfc_ncmd->cur_iocbq.cmd_flag & LPFC_IO_VMID)) {  in lpfc_nvme_prep_io_cmd()
1306 			       lpfc_ncmd->cur_iocbq.vmid_tag.cs_ctl_vmid);  in lpfc_nvme_prep_io_cmd()
1310 			wqe->words[31] = lpfc_ncmd->cur_iocbq.vmid_tag.app_id;  in lpfc_nvme_prep_io_cmd()
1334 		      struct lpfc_io_buf *lpfc_ncmd)  in lpfc_nvme_prep_io_dma()  argument
1337 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_prep_io_dma()
1338 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;  in lpfc_nvme_prep_io_dma()
1339 	struct sli4_sge *sgl = lpfc_ncmd->dma_sgl;  in lpfc_nvme_prep_io_dma()
1351 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);  in lpfc_nvme_prep_io_dma()
1365 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;  in lpfc_nvme_prep_io_dma()
1366 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {  in lpfc_nvme_prep_io_dma()
1372 					lpfc_ncmd->seg_cnt);  in lpfc_nvme_prep_io_dma()
1373 			lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1393 				lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1414 							phba, lpfc_ncmd);  in lpfc_nvme_prep_io_dma()
1417 						lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1488 		lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1533 	struct lpfc_io_buf *lpfc_ncmd;  in lpfc_nvme_fcp_io_submit()  local
1671 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, idx, expedite);  in lpfc_nvme_fcp_io_submit()
1672 	if (lpfc_ncmd == NULL) {  in lpfc_nvme_fcp_io_submit()
1683 		lpfc_ncmd->ts_cmd_start = start;  in lpfc_nvme_fcp_io_submit()
1684 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;  in lpfc_nvme_fcp_io_submit()
1686 		lpfc_ncmd->ts_cmd_start = 0;  in lpfc_nvme_fcp_io_submit()
1689 	lpfc_ncmd->rx_cmd_start = start;  in lpfc_nvme_fcp_io_submit()
1697 	freqpriv->nvme_buf = lpfc_ncmd;  in lpfc_nvme_fcp_io_submit()
1698 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;  in lpfc_nvme_fcp_io_submit()
1699 	lpfc_ncmd->ndlp = ndlp;  in lpfc_nvme_fcp_io_submit()
1700 	lpfc_ncmd->qidx = lpfc_queue_info->qidx;  in lpfc_nvme_fcp_io_submit()
1722 						&lpfc_ncmd->cur_iocbq.vmid_tag);  in lpfc_nvme_fcp_io_submit()
1724 				lpfc_ncmd->cur_iocbq.cmd_flag |= LPFC_IO_VMID;  in lpfc_nvme_fcp_io_submit()
1737 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;  in lpfc_nvme_fcp_io_submit()
1740 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);  in lpfc_nvme_fcp_io_submit()
1741 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);  in lpfc_nvme_fcp_io_submit()
1753 			 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_fcp_io_submit()
1756 	ret = lpfc_sli4_issue_wqe(phba, lpfc_ncmd->hdwq, &lpfc_ncmd->cur_iocbq);  in lpfc_nvme_fcp_io_submit()
1763 				 lpfc_ncmd->cur_iocbq.sli4_xritag);  in lpfc_nvme_fcp_io_submit()
1768 		lpfc_keep_pvt_pool_above_lowwm(phba, lpfc_ncmd->hdwq_no);  in lpfc_nvme_fcp_io_submit()
1771 	if (lpfc_ncmd->ts_cmd_start)  in lpfc_nvme_fcp_io_submit()
1772 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();  in lpfc_nvme_fcp_io_submit()
1777 		lpfc_ncmd->cpu = cpu;  in lpfc_nvme_fcp_io_submit()
1783 					lpfc_ncmd->cpu,  in lpfc_nvme_fcp_io_submit()
1790 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {  in lpfc_nvme_fcp_io_submit()
1791 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)  in lpfc_nvme_fcp_io_submit()
1797 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_fcp_io_submit()
2047 	struct lpfc_io_buf *lpfc_ncmd;  in lpfc_get_nvme_buf()  local
2053 	lpfc_ncmd = lpfc_get_io_buf(phba, NULL, idx, expedite);  in lpfc_get_nvme_buf()
2055 	if (lpfc_ncmd) {  in lpfc_get_nvme_buf()
2056 		pwqeq = &(lpfc_ncmd->cur_iocbq);  in lpfc_get_nvme_buf()
2064 		lpfc_ncmd->start_time = jiffies;  in lpfc_get_nvme_buf()
2065 		lpfc_ncmd->flags = 0;  in lpfc_get_nvme_buf()
2071 		sgl = lpfc_ncmd->dma_sgl;  in lpfc_get_nvme_buf()
2082 			lpfc_ncmd->flags |= LPFC_SBUF_BUMP_QDEPTH;  in lpfc_get_nvme_buf()
2090 	return  lpfc_ncmd;  in lpfc_get_nvme_buf()
2104 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_io_buf *lpfc_ncmd)  in lpfc_release_nvme_buf()  argument
2109 	if ((lpfc_ncmd->flags & LPFC_SBUF_BUMP_QDEPTH) && lpfc_ncmd->ndlp)  in lpfc_release_nvme_buf()
2110 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);  in lpfc_release_nvme_buf()
2112 	lpfc_ncmd->ndlp = NULL;  in lpfc_release_nvme_buf()
2113 	lpfc_ncmd->flags &= ~LPFC_SBUF_BUMP_QDEPTH;  in lpfc_release_nvme_buf()
2115 	qp = lpfc_ncmd->hdwq;  in lpfc_release_nvme_buf()
2116 	if (unlikely(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {  in lpfc_release_nvme_buf()
2120 				lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_release_nvme_buf()
2121 				lpfc_ncmd->cur_iocbq.iotag);  in lpfc_release_nvme_buf()
2124 		list_add_tail(&lpfc_ncmd->list,  in lpfc_release_nvme_buf()
2129 		lpfc_release_io_buf(phba, (struct lpfc_io_buf *)lpfc_ncmd, qp);  in lpfc_release_nvme_buf()
2672 				   struct lpfc_io_buf *lpfc_ncmd)  in lpfc_sli4_nvme_pci_offline_aborted()  argument
2679 			lpfc_ncmd->nvmeCmd,  in lpfc_sli4_nvme_pci_offline_aborted()
2680 			lpfc_ncmd->cur_iocbq.iotag);  in lpfc_sli4_nvme_pci_offline_aborted()
2686 	if (lpfc_ncmd->nvmeCmd) {  in lpfc_sli4_nvme_pci_offline_aborted()
2687 		nvme_cmd = lpfc_ncmd->nvmeCmd;  in lpfc_sli4_nvme_pci_offline_aborted()
2692 		lpfc_ncmd->nvmeCmd = NULL;  in lpfc_sli4_nvme_pci_offline_aborted()
2694 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_sli4_nvme_pci_offline_aborted()
2710 			   struct lpfc_io_buf *lpfc_ncmd)  in lpfc_sli4_nvme_xri_aborted()  argument
2714 	struct lpfc_nodelist *ndlp = lpfc_ncmd->ndlp;  in lpfc_sli4_nvme_xri_aborted()
2723 			lpfc_ncmd->nvmeCmd, xri,  in lpfc_sli4_nvme_xri_aborted()
2724 			lpfc_ncmd->cur_iocbq.iotag);  in lpfc_sli4_nvme_xri_aborted()
2730 	if (lpfc_ncmd->nvmeCmd) {  in lpfc_sli4_nvme_xri_aborted()
2731 		nvme_cmd = lpfc_ncmd->nvmeCmd;  in lpfc_sli4_nvme_xri_aborted()
2733 		lpfc_ncmd->nvmeCmd = NULL;  in lpfc_sli4_nvme_xri_aborted()
2735 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_sli4_nvme_xri_aborted()
2794 	struct lpfc_io_buf *lpfc_ncmd;  in lpfc_nvme_cancel_iocb()  local
2799 	lpfc_ncmd = pwqeIn->io_buf;  in lpfc_nvme_cancel_iocb()
2800 	if (!lpfc_ncmd) {  in lpfc_nvme_cancel_iocb()
2811 	spin_lock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_cancel_iocb()
2812 	nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_cancel_iocb()
2814 		spin_unlock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_cancel_iocb()
2815 		lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_cancel_iocb()
2818 	spin_unlock(&lpfc_ncmd->buf_lock);  in lpfc_nvme_cancel_iocb()
2822 			lpfc_ncmd->cur_iocbq.sli4_xritag);  in lpfc_nvme_cancel_iocb()