763 		       struct lpfc_nvme_buf *lpfc_ncmd,  in lpfc_nvme_adj_fcp_sgls()  argument
777 	wqe = &lpfc_ncmd->cur_iocbq.wqe;  in lpfc_nvme_adj_fcp_sgls()
786 	sgl = lpfc_ncmd->nvme_sgl;  in lpfc_nvme_adj_fcp_sgls()
860 		struct lpfc_nvme_buf *lpfc_ncmd)  in lpfc_nvme_ktime()  argument
865 	if (!lpfc_ncmd->ts_last_cmd ||  in lpfc_nvme_ktime()
866 	    !lpfc_ncmd->ts_cmd_start ||  in lpfc_nvme_ktime()
867 	    !lpfc_ncmd->ts_cmd_wqput ||  in lpfc_nvme_ktime()
868 	    !lpfc_ncmd->ts_isr_cmpl ||  in lpfc_nvme_ktime()
869 	    !lpfc_ncmd->ts_data_nvme)  in lpfc_nvme_ktime()
872 	if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_cmd_start)  in lpfc_nvme_ktime()
874 	if (lpfc_ncmd->ts_cmd_start < lpfc_ncmd->ts_last_cmd)  in lpfc_nvme_ktime()
876 	if (lpfc_ncmd->ts_cmd_wqput < lpfc_ncmd->ts_cmd_start)  in lpfc_nvme_ktime()
878 	if (lpfc_ncmd->ts_isr_cmpl < lpfc_ncmd->ts_cmd_wqput)  in lpfc_nvme_ktime()
880 	if (lpfc_ncmd->ts_data_nvme < lpfc_ncmd->ts_isr_cmpl)  in lpfc_nvme_ktime()
892 	seg1 = lpfc_ncmd->ts_cmd_start - lpfc_ncmd->ts_last_cmd;  in lpfc_nvme_ktime()
897 	seg2 = (lpfc_ncmd->ts_cmd_wqput - lpfc_ncmd->ts_cmd_start);  in lpfc_nvme_ktime()
899 	seg3 = lpfc_ncmd->ts_isr_cmpl - lpfc_ncmd->ts_cmd_start;  in lpfc_nvme_ktime()
905 	seg4 = lpfc_ncmd->ts_data_nvme - lpfc_ncmd->ts_cmd_start;  in lpfc_nvme_ktime()
932 	lpfc_ncmd->ts_last_cmd = 0;  in lpfc_nvme_ktime()
933 	lpfc_ncmd->ts_cmd_start = 0;  in lpfc_nvme_ktime()
934 	lpfc_ncmd->ts_cmd_wqput  = 0;  in lpfc_nvme_ktime()
935 	lpfc_ncmd->ts_isr_cmpl = 0;  in lpfc_nvme_ktime()
936 	lpfc_ncmd->ts_data_nvme = 0;  in lpfc_nvme_ktime()
958 	struct lpfc_nvme_buf *lpfc_ncmd =  in lpfc_nvme_io_cmd_wqe_cmpl()  local
975 	if (!lpfc_ncmd || !lpfc_ncmd->nvmeCmd || !lpfc_ncmd->nrport) {  in lpfc_nvme_io_cmd_wqe_cmpl()
976 		if (!lpfc_ncmd) {  in lpfc_nvme_io_cmd_wqe_cmpl()
987 				 lpfc_ncmd, lpfc_ncmd->nvmeCmd,  in lpfc_nvme_io_cmd_wqe_cmpl()
988 				 lpfc_ncmd->nrport);  in lpfc_nvme_io_cmd_wqe_cmpl()
991 		lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_wqe_cmpl()
994 	nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_io_cmd_wqe_cmpl()
995 	rport = lpfc_ncmd->nrport;  in lpfc_nvme_io_cmd_wqe_cmpl()
1001 			idx = lpfc_ncmd->cur_iocbq.hba_wqidx;  in lpfc_nvme_io_cmd_wqe_cmpl()
1013 			 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_wqe_cmpl()
1067 		lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_nvme_io_cmd_wqe_cmpl()
1068 		lpfc_ncmd->result = 0;  in lpfc_nvme_io_cmd_wqe_cmpl()
1072 		lpfc_ncmd->status = (status & LPFC_IOCB_STATUS_MASK);  in lpfc_nvme_io_cmd_wqe_cmpl()
1073 		lpfc_ncmd->result = (wcqe->parameter & IOERR_PARAM_MASK);  in lpfc_nvme_io_cmd_wqe_cmpl()
1086 		switch (lpfc_ncmd->status) {  in lpfc_nvme_io_cmd_wqe_cmpl()
1103 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_wqe_cmpl()
1104 					 lpfc_ncmd->status, lpfc_ncmd->result,  in lpfc_nvme_io_cmd_wqe_cmpl()
1109 			if (lpfc_ncmd->result == IOERR_ABORT_REQUESTED)  in lpfc_nvme_io_cmd_wqe_cmpl()
1115 					 lpfc_ncmd, nCmd,  in lpfc_nvme_io_cmd_wqe_cmpl()
1116 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_wqe_cmpl()
1123 					 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_io_cmd_wqe_cmpl()
1124 					 lpfc_ncmd->status, lpfc_ncmd->result,  in lpfc_nvme_io_cmd_wqe_cmpl()
1134 		lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;  in lpfc_nvme_io_cmd_wqe_cmpl()
1136 		lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;  in lpfc_nvme_io_cmd_wqe_cmpl()
1143 	if (lpfc_ncmd->ts_cmd_start) {  in lpfc_nvme_io_cmd_wqe_cmpl()
1144 		lpfc_ncmd->ts_isr_cmpl = pwqeIn->isr_timestamp;  in lpfc_nvme_io_cmd_wqe_cmpl()
1145 		lpfc_ncmd->ts_data_nvme = ktime_get_ns();  in lpfc_nvme_io_cmd_wqe_cmpl()
1146 		phba->ktime_last_cmd = lpfc_ncmd->ts_data_nvme;  in lpfc_nvme_io_cmd_wqe_cmpl()
1147 		lpfc_nvme_ktime(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_wqe_cmpl()
1150 		if (lpfc_ncmd->cpu != smp_processor_id())  in lpfc_nvme_io_cmd_wqe_cmpl()
1154 					 smp_processor_id(), lpfc_ncmd->cpu);  in lpfc_nvme_io_cmd_wqe_cmpl()
1155 		if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT)  in lpfc_nvme_io_cmd_wqe_cmpl()
1156 			phba->cpucheck_cmpl_io[lpfc_ncmd->cpu]++;  in lpfc_nvme_io_cmd_wqe_cmpl()
1164 	if (!(lpfc_ncmd->flags & LPFC_SBUF_XBUSY)) {  in lpfc_nvme_io_cmd_wqe_cmpl()
1168 		lpfc_ncmd->nvmeCmd = NULL;  in lpfc_nvme_io_cmd_wqe_cmpl()
1172 	lpfc_ncmd->nrport = NULL;  in lpfc_nvme_io_cmd_wqe_cmpl()
1176 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_io_cmd_wqe_cmpl()
1198 		      struct lpfc_nvme_buf *lpfc_ncmd,  in lpfc_nvme_prep_io_cmd()  argument
1203 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_prep_io_cmd()
1204 	struct lpfc_iocbq *pwqeq = &(lpfc_ncmd->cur_iocbq);  in lpfc_nvme_prep_io_cmd()
1228 				req_len = lpfc_ncmd->nvmeCmd->payload_length;  in lpfc_nvme_prep_io_cmd()
1304 		      struct lpfc_nvme_buf *lpfc_ncmd)  in lpfc_nvme_prep_io_dma()  argument
1307 	struct nvmefc_fcp_req *nCmd = lpfc_ncmd->nvmeCmd;  in lpfc_nvme_prep_io_dma()
1308 	union lpfc_wqe128 *wqe = &lpfc_ncmd->cur_iocbq.wqe;  in lpfc_nvme_prep_io_dma()
1309 	struct sli4_sge *sgl = lpfc_ncmd->nvme_sgl;  in lpfc_nvme_prep_io_dma()
1320 	lpfc_nvme_adj_fcp_sgls(vport, lpfc_ncmd, nCmd);  in lpfc_nvme_prep_io_dma()
1334 		lpfc_ncmd->seg_cnt = nCmd->sg_cnt;  in lpfc_nvme_prep_io_dma()
1335 		if (lpfc_ncmd->seg_cnt > lpfc_nvme_template.max_sgl_segments) {  in lpfc_nvme_prep_io_dma()
1341 					lpfc_ncmd->seg_cnt);  in lpfc_nvme_prep_io_dma()
1342 			lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1359 				lpfc_ncmd->seg_cnt = 0;  in lpfc_nvme_prep_io_dma()
1443 	struct lpfc_nvme_buf *lpfc_ncmd;  in lpfc_nvme_fcp_io_submit()  local
1561 	lpfc_ncmd = lpfc_get_nvme_buf(phba, ndlp, expedite);  in lpfc_nvme_fcp_io_submit()
1562 	if (lpfc_ncmd == NULL) {  in lpfc_nvme_fcp_io_submit()
1573 		lpfc_ncmd->ts_cmd_start = start;  in lpfc_nvme_fcp_io_submit()
1574 		lpfc_ncmd->ts_last_cmd = phba->ktime_last_cmd;  in lpfc_nvme_fcp_io_submit()
1576 		lpfc_ncmd->ts_cmd_start = 0;  in lpfc_nvme_fcp_io_submit()
1586 	freqpriv->nvme_buf = lpfc_ncmd;  in lpfc_nvme_fcp_io_submit()
1587 	lpfc_ncmd->nvmeCmd = pnvme_fcreq;  in lpfc_nvme_fcp_io_submit()
1588 	lpfc_ncmd->nrport = rport;  in lpfc_nvme_fcp_io_submit()
1589 	lpfc_ncmd->ndlp = ndlp;  in lpfc_nvme_fcp_io_submit()
1590 	lpfc_ncmd->start_time = jiffies;  in lpfc_nvme_fcp_io_submit()
1601 	lpfc_ncmd->cur_iocbq.hba_wqidx = idx;  in lpfc_nvme_fcp_io_submit()
1604 	lpfc_nvme_prep_io_cmd(vport, lpfc_ncmd, ndlp, cstat);  in lpfc_nvme_fcp_io_submit()
1605 	ret = lpfc_nvme_prep_io_dma(vport, lpfc_ncmd);  in lpfc_nvme_fcp_io_submit()
1617 			 lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_nvme_fcp_io_submit()
1620 	ret = lpfc_sli4_issue_wqe(phba, LPFC_FCP_RING, &lpfc_ncmd->cur_iocbq);  in lpfc_nvme_fcp_io_submit()
1627 				 lpfc_ncmd->cur_iocbq.sli4_xritag);  in lpfc_nvme_fcp_io_submit()
1632 	if (lpfc_ncmd->ts_cmd_start)  in lpfc_nvme_fcp_io_submit()
1633 		lpfc_ncmd->ts_cmd_wqput = ktime_get_ns();  in lpfc_nvme_fcp_io_submit()
1636 		lpfc_ncmd->cpu = smp_processor_id();  in lpfc_nvme_fcp_io_submit()
1637 		if (lpfc_ncmd->cpu != lpfc_queue_info->index) {  in lpfc_nvme_fcp_io_submit()
1644 						lpfc_ncmd->cpu,  in lpfc_nvme_fcp_io_submit()
1647 			lpfc_ncmd->cpu = lpfc_queue_info->index;  in lpfc_nvme_fcp_io_submit()
1649 		if (lpfc_ncmd->cpu < LPFC_CHECK_CPU_CNT)  in lpfc_nvme_fcp_io_submit()
1650 			phba->cpucheck_xmt_io[lpfc_ncmd->cpu]++;  in lpfc_nvme_fcp_io_submit()
1656 	if (lpfc_ncmd->nvmeCmd->sg_cnt) {  in lpfc_nvme_fcp_io_submit()
1657 		if (lpfc_ncmd->nvmeCmd->io_dir == NVMEFC_FCP_WRITE)  in lpfc_nvme_fcp_io_submit()
1663 	lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_nvme_fcp_io_submit()
1947 	struct lpfc_nvme_buf *lpfc_ncmd;  in lpfc_sli4_post_nvme_sgl_block()  local
1998 	list_for_each_entry(lpfc_ncmd, nblist, list) {  in lpfc_sli4_post_nvme_sgl_block()
2001 			cpu_to_le32(putPaddrLow(lpfc_ncmd->dma_phys_sgl));  in lpfc_sli4_post_nvme_sgl_block()
2003 			cpu_to_le32(putPaddrHigh(lpfc_ncmd->dma_phys_sgl));  in lpfc_sli4_post_nvme_sgl_block()
2005 			pdma_phys_bpl1 = lpfc_ncmd->dma_phys_sgl +  in lpfc_sli4_post_nvme_sgl_block()
2015 			xritag_start = lpfc_ncmd->cur_iocbq.sli4_xritag;  in lpfc_sli4_post_nvme_sgl_block()
2063 	struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next;  in lpfc_post_nvme_sgl_list()  local
2079 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next, post_nblist, list) {  in lpfc_post_nvme_sgl_list()
2080 		list_del_init(&lpfc_ncmd->list);  in lpfc_post_nvme_sgl_list()
2083 		    (lpfc_ncmd->cur_iocbq.sli4_xritag != last_xritag + 1)) {  in lpfc_post_nvme_sgl_list()
2088 			list_add_tail(&lpfc_ncmd->list, &prep_nblist);  in lpfc_post_nvme_sgl_list()
2092 			list_add_tail(&lpfc_ncmd->list, &prep_nblist);  in lpfc_post_nvme_sgl_list()
2101 		last_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;  in lpfc_post_nvme_sgl_list()
2113 						lpfc_ncmd->dma_phys_sgl +  in lpfc_post_nvme_sgl_list()
2117 				cur_xritag = lpfc_ncmd->cur_iocbq.sli4_xritag;  in lpfc_post_nvme_sgl_list()
2119 						lpfc_ncmd->dma_phys_sgl,  in lpfc_post_nvme_sgl_list()
2123 					lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;  in lpfc_post_nvme_sgl_list()
2126 					lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;  in lpfc_post_nvme_sgl_list()
2127 					lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_post_nvme_sgl_list()
2131 				list_add_tail(&lpfc_ncmd->list, &nvme_nblist);  in lpfc_post_nvme_sgl_list()
2152 			list_remove_head(&blck_nblist, lpfc_ncmd,  in lpfc_post_nvme_sgl_list()
2156 				lpfc_ncmd->flags |= LPFC_SBUF_XBUSY;  in lpfc_post_nvme_sgl_list()
2159 				lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;  in lpfc_post_nvme_sgl_list()
2160 				lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_post_nvme_sgl_list()
2163 			list_add_tail(&lpfc_ncmd->list, &nvme_nblist);  in lpfc_post_nvme_sgl_list()
2168 		list_remove_head(&nvme_nblist, lpfc_ncmd,  in lpfc_post_nvme_sgl_list()
2170 		lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_post_nvme_sgl_list()
2232 	struct lpfc_nvme_buf *lpfc_ncmd;  in lpfc_new_nvme_buf()  local
2246 		lpfc_ncmd = kzalloc(sizeof(struct lpfc_nvme_buf), GFP_KERNEL);  in lpfc_new_nvme_buf()
2247 		if (!lpfc_ncmd)  in lpfc_new_nvme_buf()
2254 		lpfc_ncmd->data = dma_pool_zalloc(phba->lpfc_sg_dma_buf_pool,  in lpfc_new_nvme_buf()
2256 						  &lpfc_ncmd->dma_handle);  in lpfc_new_nvme_buf()
2257 		if (!lpfc_ncmd->data) {  in lpfc_new_nvme_buf()
2258 			kfree(lpfc_ncmd);  in lpfc_new_nvme_buf()
2265 				      lpfc_ncmd->data, lpfc_ncmd->dma_handle);  in lpfc_new_nvme_buf()
2266 			kfree(lpfc_ncmd);  in lpfc_new_nvme_buf()
2269 		pwqeq = &(lpfc_ncmd->cur_iocbq);  in lpfc_new_nvme_buf()
2276 				      lpfc_ncmd->data, lpfc_ncmd->dma_handle);  in lpfc_new_nvme_buf()
2277 			kfree(lpfc_ncmd);  in lpfc_new_nvme_buf()
2287 		pwqeq->context1 = lpfc_ncmd;  in lpfc_new_nvme_buf()
2291 		lpfc_ncmd->nvme_sgl = lpfc_ncmd->data;  in lpfc_new_nvme_buf()
2292 		sgl = lpfc_ncmd->nvme_sgl;  in lpfc_new_nvme_buf()
2293 		pdma_phys_sgl = lpfc_ncmd->dma_handle;  in lpfc_new_nvme_buf()
2294 		lpfc_ncmd->dma_phys_sgl = pdma_phys_sgl;  in lpfc_new_nvme_buf()
2305 		lpfc_ncmd->cur_iocbq.context1 = lpfc_ncmd;  in lpfc_new_nvme_buf()
2311 		list_add_tail(&lpfc_ncmd->list, &post_nblist);  in lpfc_new_nvme_buf()
2333 	struct lpfc_nvme_buf *lpfc_ncmd, *lpfc_ncmd_next;  in lpfc_nvme_buf()  local
2335 	list_for_each_entry_safe(lpfc_ncmd, lpfc_ncmd_next,  in lpfc_nvme_buf()
2337 		list_del_init(&lpfc_ncmd->list);  in lpfc_nvme_buf()
2339 		return lpfc_ncmd;  in lpfc_nvme_buf()
2359 	struct lpfc_nvme_buf *lpfc_ncmd = NULL;  in lpfc_get_nvme_buf()  local
2364 		lpfc_ncmd = lpfc_nvme_buf(phba);  in lpfc_get_nvme_buf()
2365 	if (!lpfc_ncmd) {  in lpfc_get_nvme_buf()
2374 			lpfc_ncmd = lpfc_nvme_buf(phba);  in lpfc_get_nvme_buf()
2378 	if (lpfc_ndlp_check_qdepth(phba, ndlp) && lpfc_ncmd) {  in lpfc_get_nvme_buf()
2380 		lpfc_ncmd->flags |= LPFC_BUMP_QDEPTH;  in lpfc_get_nvme_buf()
2382 	return  lpfc_ncmd;  in lpfc_get_nvme_buf()
2396 lpfc_release_nvme_buf(struct lpfc_hba *phba, struct lpfc_nvme_buf *lpfc_ncmd)  in lpfc_release_nvme_buf()  argument
2400 	if ((lpfc_ncmd->flags & LPFC_BUMP_QDEPTH) && lpfc_ncmd->ndlp)  in lpfc_release_nvme_buf()
2401 		atomic_dec(&lpfc_ncmd->ndlp->cmd_pending);  in lpfc_release_nvme_buf()
2403 	lpfc_ncmd->nonsg_phys = 0;  in lpfc_release_nvme_buf()
2404 	lpfc_ncmd->ndlp = NULL;  in lpfc_release_nvme_buf()
2405 	lpfc_ncmd->flags &= ~LPFC_BUMP_QDEPTH;  in lpfc_release_nvme_buf()
2407 	if (lpfc_ncmd->flags & LPFC_SBUF_XBUSY) {  in lpfc_release_nvme_buf()
2411 				lpfc_ncmd->cur_iocbq.sli4_xritag,  in lpfc_release_nvme_buf()
2412 				lpfc_ncmd->cur_iocbq.iotag);  in lpfc_release_nvme_buf()
2416 		list_add_tail(&lpfc_ncmd->list,  in lpfc_release_nvme_buf()
2421 		lpfc_ncmd->nvmeCmd = NULL;  in lpfc_release_nvme_buf()
2422 		lpfc_ncmd->cur_iocbq.iocb_flag = LPFC_IO_NVME;  in lpfc_release_nvme_buf()
2424 		list_add_tail(&lpfc_ncmd->list, &phba->lpfc_nvme_buf_list_put);  in lpfc_release_nvme_buf()
2924 	struct lpfc_nvme_buf *lpfc_ncmd, *next_lpfc_ncmd;  in lpfc_sli4_nvme_xri_aborted()  local
2933 	list_for_each_entry_safe(lpfc_ncmd, next_lpfc_ncmd,  in lpfc_sli4_nvme_xri_aborted()
2936 		if (lpfc_ncmd->cur_iocbq.sli4_xritag == xri) {  in lpfc_sli4_nvme_xri_aborted()
2937 			list_del_init(&lpfc_ncmd->list);  in lpfc_sli4_nvme_xri_aborted()
2938 			lpfc_ncmd->flags &= ~LPFC_SBUF_XBUSY;  in lpfc_sli4_nvme_xri_aborted()
2939 			lpfc_ncmd->status = IOSTAT_SUCCESS;  in lpfc_sli4_nvme_xri_aborted()
2944 			ndlp = lpfc_ncmd->ndlp;  in lpfc_sli4_nvme_xri_aborted()
2951 					lpfc_ncmd->nvmeCmd, xri,  in lpfc_sli4_nvme_xri_aborted()
2952 					lpfc_ncmd->cur_iocbq.iotag);  in lpfc_sli4_nvme_xri_aborted()
2958 			if (lpfc_ncmd->nvmeCmd) {  in lpfc_sli4_nvme_xri_aborted()
2959 				nvme_cmd = lpfc_ncmd->nvmeCmd;  in lpfc_sli4_nvme_xri_aborted()
2961 				lpfc_ncmd->nvmeCmd = NULL;  in lpfc_sli4_nvme_xri_aborted()
2963 			lpfc_release_nvme_buf(phba, lpfc_ncmd);  in lpfc_sli4_nvme_xri_aborted()