659 	struct srpt_ioctx *ioctx;  in srpt_alloc_ioctx()  local
661 	ioctx = kzalloc(ioctx_size, GFP_KERNEL);  in srpt_alloc_ioctx()
662 	if (!ioctx)  in srpt_alloc_ioctx()
665 	ioctx->buf = kmem_cache_alloc(buf_cache, GFP_KERNEL);  in srpt_alloc_ioctx()
666 	if (!ioctx->buf)  in srpt_alloc_ioctx()
669 	ioctx->dma = ib_dma_map_single(sdev->device, ioctx->buf,  in srpt_alloc_ioctx()
671 	if (ib_dma_mapping_error(sdev->device, ioctx->dma))  in srpt_alloc_ioctx()
674 	return ioctx;  in srpt_alloc_ioctx()
677 	kmem_cache_free(buf_cache, ioctx->buf);  in srpt_alloc_ioctx()
679 	kfree(ioctx);  in srpt_alloc_ioctx()
691 static void srpt_free_ioctx(struct srpt_device *sdev, struct srpt_ioctx *ioctx,  in srpt_free_ioctx()  argument
695 	if (!ioctx)  in srpt_free_ioctx()
698 	ib_dma_unmap_single(sdev->device, ioctx->dma,  in srpt_free_ioctx()
700 	kmem_cache_free(buf_cache, ioctx->buf);  in srpt_free_ioctx()
701 	kfree(ioctx);  in srpt_free_ioctx()
778 static enum srpt_command_state srpt_set_cmd_state(struct srpt_send_ioctx *ioctx,  in srpt_set_cmd_state()  argument
783 	previous = ioctx->state;  in srpt_set_cmd_state()
785 		ioctx->state = new;  in srpt_set_cmd_state()
798 static bool srpt_test_and_set_cmd_state(struct srpt_send_ioctx *ioctx,  in srpt_test_and_set_cmd_state()  argument
804 	WARN_ON(!ioctx);  in srpt_test_and_set_cmd_state()
808 	previous = ioctx->state;  in srpt_test_and_set_cmd_state()
810 		ioctx->state = new;  in srpt_test_and_set_cmd_state()
822 			  struct srpt_recv_ioctx *ioctx)  in srpt_post_recv()  argument
828 	list.addr = ioctx->ioctx.dma + ioctx->ioctx.offset;  in srpt_post_recv()
832 	ioctx->ioctx.cqe.done = srpt_recv_done;  in srpt_post_recv()
833 	wr.wr_cqe = &ioctx->ioctx.cqe;  in srpt_post_recv()
888 static int srpt_alloc_rw_ctxs(struct srpt_send_ioctx *ioctx,  in srpt_alloc_rw_ctxs()  argument
892 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);  in srpt_alloc_rw_ctxs()
893 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_alloc_rw_ctxs()
899 		ioctx->rw_ctxs = &ioctx->s_rw_ctx;  in srpt_alloc_rw_ctxs()
901 		ioctx->rw_ctxs = kmalloc_array(nbufs, sizeof(*ioctx->rw_ctxs),  in srpt_alloc_rw_ctxs()
903 		if (!ioctx->rw_ctxs)  in srpt_alloc_rw_ctxs()
907 	for (i = ioctx->n_rw_ctx; i < nbufs; i++, db++) {  in srpt_alloc_rw_ctxs()
908 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];  in srpt_alloc_rw_ctxs()
925 		ioctx->n_rdma += ret;  in srpt_alloc_rw_ctxs()
926 		ioctx->n_rw_ctx++;  in srpt_alloc_rw_ctxs()
945 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];  in srpt_alloc_rw_ctxs()
951 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)  in srpt_alloc_rw_ctxs()
952 		kfree(ioctx->rw_ctxs);  in srpt_alloc_rw_ctxs()
957 				    struct srpt_send_ioctx *ioctx)  in srpt_free_rw_ctxs()  argument
959 	enum dma_data_direction dir = target_reverse_dma_direction(&ioctx->cmd);  in srpt_free_rw_ctxs()
962 	for (i = 0; i < ioctx->n_rw_ctx; i++) {  in srpt_free_rw_ctxs()
963 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];  in srpt_free_rw_ctxs()
970 	if (ioctx->rw_ctxs != &ioctx->s_rw_ctx)  in srpt_free_rw_ctxs()
971 		kfree(ioctx->rw_ctxs);  in srpt_free_rw_ctxs()
1012 		struct srpt_send_ioctx *ioctx,  in srpt_get_desc_tbl()  argument
1035 	ioctx->cmd.data_direction = *dir;  in srpt_get_desc_tbl()
1042 		return srpt_alloc_rw_ctxs(ioctx, db, 1, sg, sg_cnt);  in srpt_get_desc_tbl()
1060 		return srpt_alloc_rw_ctxs(ioctx, idb->desc_list, nbufs,  in srpt_get_desc_tbl()
1087 		ioctx->recv_ioctx = recv_ioctx;  in srpt_get_desc_tbl()
1092 		sg_init_one(&ioctx->imm_sg, data, len);  in srpt_get_desc_tbl()
1093 		*sg = &ioctx->imm_sg;  in srpt_get_desc_tbl()
1219 	struct srpt_send_ioctx *ioctx;  in srpt_get_send_ioctx()  local
1228 	ioctx = ch->ioctx_ring[tag];  in srpt_get_send_ioctx()
1229 	BUG_ON(ioctx->ch != ch);  in srpt_get_send_ioctx()
1230 	ioctx->state = SRPT_STATE_NEW;  in srpt_get_send_ioctx()
1231 	WARN_ON_ONCE(ioctx->recv_ioctx);  in srpt_get_send_ioctx()
1232 	ioctx->n_rdma = 0;  in srpt_get_send_ioctx()
1233 	ioctx->n_rw_ctx = 0;  in srpt_get_send_ioctx()
1234 	ioctx->queue_status_only = false;  in srpt_get_send_ioctx()
1239 	memset(&ioctx->cmd, 0, sizeof(ioctx->cmd));  in srpt_get_send_ioctx()
1240 	memset(&ioctx->sense_data, 0, sizeof(ioctx->sense_data));  in srpt_get_send_ioctx()
1241 	ioctx->cmd.map_tag = tag;  in srpt_get_send_ioctx()
1242 	ioctx->cmd.map_cpu = cpu;  in srpt_get_send_ioctx()
1244 	return ioctx;  in srpt_get_send_ioctx()
1251 static int srpt_abort_cmd(struct srpt_send_ioctx *ioctx)  in srpt_abort_cmd()  argument
1255 	BUG_ON(!ioctx);  in srpt_abort_cmd()
1262 	state = ioctx->state;  in srpt_abort_cmd()
1265 		ioctx->state = SRPT_STATE_DATA_IN;  in srpt_abort_cmd()
1269 		ioctx->state = SRPT_STATE_DONE;  in srpt_abort_cmd()
1278 		 ioctx->state, ioctx->cmd.tag);  in srpt_abort_cmd()
1291 		pr_debug("tag %#llx: RDMA read error\n", ioctx->cmd.tag);  in srpt_abort_cmd()
1292 		transport_generic_request_failure(&ioctx->cmd,  in srpt_abort_cmd()
1300 		transport_generic_free_cmd(&ioctx->cmd, 0);  in srpt_abort_cmd()
1303 		transport_generic_free_cmd(&ioctx->cmd, 0);  in srpt_abort_cmd()
1326 	struct srpt_send_ioctx *ioctx =  in srpt_rdma_read_done()  local
1329 	WARN_ON(ioctx->n_rdma <= 0);  in srpt_rdma_read_done()
1330 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_rdma_read_done()
1331 	ioctx->n_rdma = 0;  in srpt_rdma_read_done()
1335 			ioctx, wc->status);  in srpt_rdma_read_done()
1336 		srpt_abort_cmd(ioctx);  in srpt_rdma_read_done()
1340 	if (srpt_test_and_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA,  in srpt_rdma_read_done()
1342 		target_execute_cmd(&ioctx->cmd);  in srpt_rdma_read_done()
1345 		       __LINE__, ioctx->state);  in srpt_rdma_read_done()
1364 			      struct srpt_send_ioctx *ioctx, u64 tag,  in srpt_build_cmd_rsp()  argument
1377 	srp_rsp = ioctx->ioctx.buf;  in srpt_build_cmd_rsp()
1380 	sense_data = ioctx->sense_data;  in srpt_build_cmd_rsp()
1381 	sense_data_len = ioctx->cmd.scsi_sense_length;  in srpt_build_cmd_rsp()
1382 	WARN_ON(sense_data_len > sizeof(ioctx->sense_data));  in srpt_build_cmd_rsp()
1422 				  struct srpt_send_ioctx *ioctx,  in srpt_build_tskmgmt_rsp()  argument
1432 	srp_rsp = ioctx->ioctx.buf;  in srpt_build_tskmgmt_rsp()
1450 	struct srpt_send_ioctx *ioctx = container_of(cmd,  in srpt_check_stop_free()  local
1453 	return target_put_sess_cmd(&ioctx->cmd);  in srpt_check_stop_free()
1476 	srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;  in srpt_handle_cmd()
1563 	srp_tsk = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;  in srpt_handle_tsk_mgmt()
1604 				   recv_ioctx->ioctx.dma,  in srpt_handle_new_iu()
1605 				   recv_ioctx->ioctx.offset + srp_max_req_size,  in srpt_handle_new_iu()
1608 	srp_cmd = recv_ioctx->ioctx.buf + recv_ioctx->ioctx.offset;  in srpt_handle_new_iu()
1663 	struct srpt_recv_ioctx *ioctx =  in srpt_recv_done()  local
1664 		container_of(wc->wr_cqe, struct srpt_recv_ioctx, ioctx.cqe);  in srpt_recv_done()
1672 		ioctx->byte_len = wc->byte_len;  in srpt_recv_done()
1673 		srpt_handle_new_iu(ch, ioctx);  in srpt_recv_done()
1676 				    ioctx, wc->status);  in srpt_recv_done()
1724 	struct srpt_send_ioctx *ioctx =  in srpt_send_done()  local
1725 		container_of(wc->wr_cqe, struct srpt_send_ioctx, ioctx.cqe);  in srpt_send_done()
1728 	state = srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);  in srpt_send_done()
1733 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_send_done()
1737 			ioctx, wc->status);  in srpt_send_done()
1740 		transport_generic_free_cmd(&ioctx->cmd, 0);  in srpt_send_done()
1743 		       ioctx->ioctx.index);  in srpt_send_done()
2702 	struct srpt_send_ioctx *ioctx =  in srpt_write_pending()  local
2704 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_write_pending()
2706 	struct ib_cqe *cqe = &ioctx->rdma_cqe;  in srpt_write_pending()
2710 	if (ioctx->recv_ioctx) {  in srpt_write_pending()
2711 		srpt_set_cmd_state(ioctx, SRPT_STATE_DATA_IN);  in srpt_write_pending()
2712 		target_execute_cmd(&ioctx->cmd);  in srpt_write_pending()
2716 	new_state = srpt_set_cmd_state(ioctx, SRPT_STATE_NEED_DATA);  in srpt_write_pending()
2719 	if (atomic_sub_return(ioctx->n_rdma, &ch->sq_wr_avail) < 0) {  in srpt_write_pending()
2721 				__func__, ioctx->n_rdma);  in srpt_write_pending()
2727 	for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {  in srpt_write_pending()
2728 		struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];  in srpt_write_pending()
2738 			 __func__, ret, ioctx->n_rdma,  in srpt_write_pending()
2745 	atomic_add(ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_write_pending()
2769 	struct srpt_send_ioctx *ioctx =  in srpt_queue_response()  local
2771 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_queue_response()
2781 	state = ioctx->state;  in srpt_queue_response()
2785 		ioctx->state = SRPT_STATE_CMD_RSP_SENT;  in srpt_queue_response()
2788 		ioctx->state = SRPT_STATE_MGMT_RSP_SENT;  in srpt_queue_response()
2792 			ch, ioctx->ioctx.index, ioctx->state);  in srpt_queue_response()
2800 	if (ioctx->cmd.data_direction == DMA_FROM_DEVICE &&  in srpt_queue_response()
2801 	    ioctx->cmd.data_length &&  in srpt_queue_response()
2802 	    !ioctx->queue_status_only) {  in srpt_queue_response()
2803 		for (i = ioctx->n_rw_ctx - 1; i >= 0; i--) {  in srpt_queue_response()
2804 			struct srpt_rw_ctx *ctx = &ioctx->rw_ctxs[i];  in srpt_queue_response()
2812 		resp_len = srpt_build_cmd_rsp(ch, ioctx, ioctx->cmd.tag,  in srpt_queue_response()
2817 		resp_len = srpt_build_tskmgmt_rsp(ch, ioctx, srp_tm_status,  in srpt_queue_response()
2818 						 ioctx->cmd.tag);  in srpt_queue_response()
2823 	if (unlikely(atomic_sub_return(1 + ioctx->n_rdma,  in srpt_queue_response()
2826 				__func__, ioctx->n_rdma);  in srpt_queue_response()
2831 	ib_dma_sync_single_for_device(sdev->device, ioctx->ioctx.dma, resp_len,  in srpt_queue_response()
2834 	sge.addr = ioctx->ioctx.dma;  in srpt_queue_response()
2838 	ioctx->ioctx.cqe.done = srpt_send_done;  in srpt_queue_response()
2840 	send_wr.wr_cqe = &ioctx->ioctx.cqe;  in srpt_queue_response()
2849 			__func__, ioctx->cmd.tag, ret);  in srpt_queue_response()
2856 	atomic_add(1 + ioctx->n_rdma, &ch->sq_wr_avail);  in srpt_queue_response()
2858 	srpt_set_cmd_state(ioctx, SRPT_STATE_DONE);  in srpt_queue_response()
2859 	target_put_sess_cmd(&ioctx->cmd);  in srpt_queue_response()
2881 	struct srpt_send_ioctx *ioctx = container_of(cmd,  in srpt_aborted_task()  local
2883 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_aborted_task()
2890 	struct srpt_send_ioctx *ioctx;  in srpt_queue_status()  local
2892 	ioctx = container_of(cmd, struct srpt_send_ioctx, cmd);  in srpt_queue_status()
2893 	BUG_ON(ioctx->sense_data != cmd->sense_buffer);  in srpt_queue_status()
2897 	ioctx->queue_status_only = true;  in srpt_queue_status()
3283 	struct srpt_send_ioctx *ioctx = container_of(se_cmd,  in srpt_release_cmd()  local
3285 	struct srpt_rdma_ch *ch = ioctx->ch;  in srpt_release_cmd()
3286 	struct srpt_recv_ioctx *recv_ioctx = ioctx->recv_ioctx;  in srpt_release_cmd()
3288 	WARN_ON_ONCE(ioctx->state != SRPT_STATE_DONE &&  in srpt_release_cmd()
3289 		     !(ioctx->cmd.transport_state & CMD_T_ABORTED));  in srpt_release_cmd()
3293 		ioctx->recv_ioctx = NULL;  in srpt_release_cmd()
3297 	if (ioctx->n_rw_ctx) {  in srpt_release_cmd()
3298 		srpt_free_rw_ctxs(ch, ioctx);  in srpt_release_cmd()
3299 		ioctx->n_rw_ctx = 0;  in srpt_release_cmd()
3341 	struct srpt_send_ioctx *ioctx;  in srpt_get_tcm_cmd_state()  local
3343 	ioctx = container_of(se_cmd, struct srpt_send_ioctx, cmd);  in srpt_get_tcm_cmd_state()
3344 	return ioctx->state;  in srpt_get_tcm_cmd_state()