423 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_init_request()  local
428 	iod->nvmeq = nvmeq;  in nvme_init_request()
466 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_pci_iod_list()  local
467 	return (void **)(iod->sg + blk_rq_nr_phys_segments(req));  in nvme_pci_iod_list()
472 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_pci_use_sgls()  local
483 	if (!iod->nvmeq->qid)  in nvme_pci_use_sgls()
492 	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);  in nvme_init_iod()  local
496 	iod->use_sgl = nvme_pci_use_sgls(dev, rq);  in nvme_init_iod()
499 		iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);  in nvme_init_iod()
500 		if (!iod->sg)  in nvme_init_iod()
503 		iod->sg = iod->inline_sg;  in nvme_init_iod()
506 	iod->aborted = 0;  in nvme_init_iod()
507 	iod->npages = -1;  in nvme_init_iod()
508 	iod->nents = 0;  in nvme_init_iod()
509 	iod->length = size;  in nvme_init_iod()
516 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_free_iod()  local
518 	dma_addr_t dma_addr = iod->first_dma, next_dma_addr;  in nvme_free_iod()
522 	if (iod->npages == 0)  in nvme_free_iod()
526 	for (i = 0; i < iod->npages; i++) {  in nvme_free_iod()
529 		if (iod->use_sgl) {  in nvme_free_iod()
544 	if (iod->sg != iod->inline_sg)  in nvme_free_iod()
545 		mempool_free(iod->sg, dev->iod_mempool);  in nvme_free_iod()
565 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_pci_setup_prps()  local
568 	struct scatterlist *sg = iod->sg;  in nvme_pci_setup_prps()
580 		iod->first_dma = 0;  in nvme_pci_setup_prps()
594 		iod->first_dma = dma_addr;  in nvme_pci_setup_prps()
601 		iod->npages = 0;  in nvme_pci_setup_prps()
604 		iod->npages = 1;  in nvme_pci_setup_prps()
609 		iod->first_dma = dma_addr;  in nvme_pci_setup_prps()
610 		iod->npages = -1;  in nvme_pci_setup_prps()
614 	iod->first_dma = prp_dma;  in nvme_pci_setup_prps()
622 			list[iod->npages++] = prp_list;  in nvme_pci_setup_prps()
643 	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));  in nvme_pci_setup_prps()
644 	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);  in nvme_pci_setup_prps()
649 	WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),  in nvme_pci_setup_prps()
651 			blk_rq_payload_bytes(req), iod->nents);  in nvme_pci_setup_prps()
679 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_pci_setup_sgls()  local
682 	struct scatterlist *sg = iod->sg;  in nvme_pci_setup_sgls()
696 		iod->npages = 0;  in nvme_pci_setup_sgls()
699 		iod->npages = 1;  in nvme_pci_setup_sgls()
704 		iod->npages = -1;  in nvme_pci_setup_sgls()
709 	iod->first_dma = sgl_dma;  in nvme_pci_setup_sgls()
723 			nvme_pci_iod_list(req)[iod->npages++] = sg_list;  in nvme_pci_setup_sgls()
738 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_map_data()  local
745 	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));  in nvme_map_data()
746 	iod->nents = blk_rq_map_sg(q, req, iod->sg);  in nvme_map_data()
747 	if (!iod->nents)  in nvme_map_data()
751 	nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,  in nvme_map_data()
756 	if (iod->use_sgl)  in nvme_map_data()
769 		sg_init_table(&iod->meta_sg, 1);  in nvme_map_data()
770 		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)  in nvme_map_data()
773 		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))  in nvme_map_data()
778 		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));  in nvme_map_data()
782 	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);  in nvme_map_data()
789 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_unmap_data()  local
793 	if (iod->nents) {  in nvme_unmap_data()
794 		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);  in nvme_unmap_data()
796 			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);  in nvme_unmap_data()
849 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_pci_complete_rq()  local
851 	nvme_unmap_data(iod->nvmeq->dev, req);  in nvme_pci_complete_rq()
1073 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in abort_endio()  local
1074 	struct nvme_queue *nvmeq = iod->nvmeq;  in abort_endio()
1128 	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);  in nvme_timeout()  local
1129 	struct nvme_queue *nvmeq = iod->nvmeq;  in nvme_timeout()
1186 	if (!nvmeq->qid || iod->aborted) {  in nvme_timeout()
1201 	iod->aborted = 1;  in nvme_timeout()