362 			     struct skd_request_context *skreq);
364 			       struct skd_request_context *skreq);
378 			  struct skd_request_context *skreq, const char *event);
424 			    struct skd_request_context *skreq)  in skd_prep_zerosize_flush_cdb()  argument
426 	skreq->flush_cmd = 1;  in skd_prep_zerosize_flush_cdb()
487 	struct skd_request_context *const skreq = blk_mq_rq_to_pdu(req);  in skd_mq_queue_rq()  local
502 	SKD_ASSERT(skreq->state == SKD_REQ_STATE_IDLE);  in skd_mq_queue_rq()
508 	skreq->id = tag + SKD_ID_RW_REQUEST;  in skd_mq_queue_rq()
509 	skreq->flush_cmd = 0;  in skd_mq_queue_rq()
510 	skreq->n_sg = 0;  in skd_mq_queue_rq()
511 	skreq->sg_byte_count = 0;  in skd_mq_queue_rq()
513 	skreq->fitmsg_id = 0;  in skd_mq_queue_rq()
515 	skreq->data_dir = data_dir == READ ? DMA_FROM_DEVICE : DMA_TO_DEVICE;  in skd_mq_queue_rq()
517 	if (req->bio && !skd_preop_sg_list(skdev, skreq)) {  in skd_mq_queue_rq()
519 		skreq->status = BLK_STS_RESOURCE;  in skd_mq_queue_rq()
524 	dma_sync_single_for_device(&skdev->pdev->dev, skreq->sksg_dma_address,  in skd_mq_queue_rq()
525 				   skreq->n_sg *  in skd_mq_queue_rq()
549 	skreq->fitmsg_id = skmsg->id;  in skd_mq_queue_rq()
554 	scsi_req->hdr.tag = skreq->id;  in skd_mq_queue_rq()
556 		cpu_to_be64(skreq->sksg_dma_address);  in skd_mq_queue_rq()
559 		skd_prep_zerosize_flush_cdb(scsi_req, skreq);  in skd_mq_queue_rq()
560 		SKD_ASSERT(skreq->flush_cmd == 1);  in skd_mq_queue_rq()
568 	scsi_req->hdr.sg_list_len_bytes = cpu_to_be32(skreq->sg_byte_count);  in skd_mq_queue_rq()
571 	skreq->state = SKD_REQ_STATE_BUSY;  in skd_mq_queue_rq()
576 	dev_dbg(&skdev->pdev->dev, "req=0x%x busy=%d\n", skreq->id,  in skd_mq_queue_rq()
609 	struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);  in skd_complete_rq()  local
611 	blk_mq_end_request(req, skreq->status);  in skd_complete_rq()
615 			     struct skd_request_context *skreq)  in skd_preop_sg_list()  argument
617 	struct request *req = blk_mq_rq_from_pdu(skreq);  in skd_preop_sg_list()
618 	struct scatterlist *sgl = &skreq->sg[0], *sg;  in skd_preop_sg_list()
622 	skreq->sg_byte_count = 0;  in skd_preop_sg_list()
624 	WARN_ON_ONCE(skreq->data_dir != DMA_TO_DEVICE &&  in skd_preop_sg_list()
625 		     skreq->data_dir != DMA_FROM_DEVICE);  in skd_preop_sg_list()
635 	n_sg = pci_map_sg(skdev->pdev, sgl, n_sg, skreq->data_dir);  in skd_preop_sg_list()
641 	skreq->n_sg = n_sg;  in skd_preop_sg_list()
644 		struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];  in skd_preop_sg_list()
650 		skreq->sg_byte_count += cnt;  in skd_preop_sg_list()
655 	skreq->sksg_list[n_sg - 1].next_desc_ptr = 0LL;  in skd_preop_sg_list()
656 	skreq->sksg_list[n_sg - 1].control = FIT_SGD_CONTROL_LAST;  in skd_preop_sg_list()
661 			skreq->id, skreq->sksg_list, &skreq->sksg_dma_address);  in skd_preop_sg_list()
663 			struct fit_sg_descriptor *sgd = &skreq->sksg_list[i];  in skd_preop_sg_list()
676 			       struct skd_request_context *skreq)  in skd_postop_sg_list()  argument
682 	skreq->sksg_list[skreq->n_sg - 1].next_desc_ptr =  in skd_postop_sg_list()
683 		skreq->sksg_dma_address +  in skd_postop_sg_list()
684 		((skreq->n_sg) * sizeof(struct fit_sg_descriptor));  in skd_postop_sg_list()
685 	pci_unmap_sg(skdev->pdev, &skreq->sg[0], skreq->n_sg, skreq->data_dir);  in skd_postop_sg_list()
1405 				      struct skd_request_context *skreq,  in skd_resolve_req_exception()  argument
1408 	u8 cmp_status = skreq->completion.status;  in skd_resolve_req_exception()
1410 	switch (skd_check_status(skdev, cmp_status, &skreq->err_info)) {  in skd_resolve_req_exception()
1413 		skreq->status = BLK_STS_OK;  in skd_resolve_req_exception()
1418 		skd_log_skreq(skdev, skreq, "retry(busy)");  in skd_resolve_req_exception()
1428 			skd_log_skreq(skdev, skreq, "retry");  in skd_resolve_req_exception()
1436 		skreq->status = BLK_STS_IOERR;  in skd_resolve_req_exception()
1443 			      struct skd_request_context *skreq)  in skd_release_skreq()  argument
1448 	skreq->state = SKD_REQ_STATE_IDLE;  in skd_release_skreq()
1460 	struct skd_request_context *skreq;  in skd_isr_completion_posted()  local
1524 		skreq = blk_mq_rq_to_pdu(rq);  in skd_isr_completion_posted()
1529 		if (skreq->id != req_id) {  in skd_isr_completion_posted()
1532 				req_id, skreq->id, cmp_cntxt);  in skd_isr_completion_posted()
1537 		SKD_ASSERT(skreq->state == SKD_REQ_STATE_BUSY);  in skd_isr_completion_posted()
1539 		skreq->completion = *skcmp;  in skd_isr_completion_posted()
1541 			skreq->err_info = *skerr;  in skd_isr_completion_posted()
1547 		if (skreq->n_sg > 0)  in skd_isr_completion_posted()
1548 			skd_postop_sg_list(skdev, skreq);  in skd_isr_completion_posted()
1550 		skd_release_skreq(skdev, skreq);  in skd_isr_completion_posted()
1556 			skreq->status = BLK_STS_OK;  in skd_isr_completion_posted()
1559 			skd_resolve_req_exception(skdev, skreq, rq);  in skd_isr_completion_posted()
1892 	struct skd_request_context *skreq = blk_mq_rq_to_pdu(req);  in skd_recover_request()  local
1894 	if (skreq->state != SKD_REQ_STATE_BUSY)  in skd_recover_request()
1897 	skd_log_skreq(skdev, skreq, "recover");  in skd_recover_request()
1900 	if (skreq->n_sg > 0)  in skd_recover_request()
1901 		skd_postop_sg_list(skdev, skreq);  in skd_recover_request()
1903 	skreq->state = SKD_REQ_STATE_IDLE;  in skd_recover_request()
1904 	skreq->status = BLK_STS_IOERR;  in skd_recover_request()
2737 	struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);  in skd_init_request()  local
2739 	skreq->state = SKD_REQ_STATE_IDLE;  in skd_init_request()
2740 	skreq->sg = (void *)(skreq + 1);  in skd_init_request()
2741 	sg_init_table(skreq->sg, skd_sgs_per_request);  in skd_init_request()
2742 	skreq->sksg_list = skd_cons_sg_list(skdev, skd_sgs_per_request,  in skd_init_request()
2743 					    &skreq->sksg_dma_address);  in skd_init_request()
2745 	return skreq->sksg_list ? 0 : -ENOMEM;  in skd_init_request()
2752 	struct skd_request_context *skreq = blk_mq_rq_to_pdu(rq);  in skd_exit_request()  local
2754 	skd_free_sg_list(skdev, skreq->sksg_list, skreq->sksg_dma_address);  in skd_exit_request()
3587 			  struct skd_request_context *skreq, const char *event)  in skd_log_skreq()  argument
3589 	struct request *req = blk_mq_rq_from_pdu(skreq);  in skd_log_skreq()
3593 	dev_dbg(&skdev->pdev->dev, "skreq=%p event='%s'\n", skreq, event);  in skd_log_skreq()
3595 		skd_skreq_state_to_str(skreq->state), skreq->state, skreq->id,  in skd_log_skreq()
3596 		skreq->fitmsg_id);  in skd_log_skreq()
3598 		skreq->data_dir, skreq->n_sg);  in skd_log_skreq()