170 	struct vhost_virtqueue vq;  member
251 	struct vhost_virtqueue *vq;  in vhost_scsi_init_inflight()  local
255 		vq = &vs->vqs[i].vq;  in vhost_scsi_init_inflight()
257 		mutex_lock(&vq->mutex);  in vhost_scsi_init_inflight()
270 		mutex_unlock(&vq->mutex);  in vhost_scsi_init_inflight()
275 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)  in vhost_scsi_get_inflight()  argument
280 	svq = container_of(vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_get_inflight()
336 				struct vhost_scsi_virtqueue, vq);  in vhost_scsi_release_cmd_res()
439 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_allocate_evt()  local
449 		vq_err(vq, "Failed to allocate vhost_scsi_evt\n");  in vhost_scsi_allocate_evt()
454 	evt->event.event = cpu_to_vhost32(vq, event);  in vhost_scsi_allocate_evt()
455 	evt->event.reason = cpu_to_vhost32(vq, reason);  in vhost_scsi_allocate_evt()
469 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_evt_work()  local
475 	if (!vhost_vq_get_backend(vq)) {  in vhost_scsi_do_evt_work()
481 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_do_evt_work()
482 	head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_do_evt_work()
483 			ARRAY_SIZE(vq->iov), &out, &in,  in vhost_scsi_do_evt_work()
489 	if (head == vq->num) {  in vhost_scsi_do_evt_work()
490 		if (vhost_enable_notify(&vs->dev, vq))  in vhost_scsi_do_evt_work()
496 	if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {  in vhost_scsi_do_evt_work()
497 		vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",  in vhost_scsi_do_evt_work()
498 				vq->iov[out].iov_len);  in vhost_scsi_do_evt_work()
504 		event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);  in vhost_scsi_do_evt_work()
508 	eventp = vq->iov[out].iov_base;  in vhost_scsi_do_evt_work()
511 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_do_evt_work()
513 		vq_err(vq, "Faulted on vhost_scsi_send_event\n");  in vhost_scsi_do_evt_work()
520 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_evt_work()  local
524 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_work()
530 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_work()
547 	int ret, vq;  in vhost_scsi_complete_cmd_work()  local
572 			q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_complete_cmd_work()
573 			vq = q - vs->vqs;  in vhost_scsi_complete_cmd_work()
574 			__set_bit(vq, vs->compl_bitmap);  in vhost_scsi_complete_cmd_work()
581 	vq = -1;  in vhost_scsi_complete_cmd_work()
582 	while ((vq = find_next_bit(vs->compl_bitmap, vs->dev.nvqs, vq + 1))  in vhost_scsi_complete_cmd_work()
584 		vhost_signal(&vs->dev, &vs->vqs[vq].vq);  in vhost_scsi_complete_cmd_work()
588 vhost_scsi_get_cmd(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,  in vhost_scsi_get_cmd()  argument
592 	struct vhost_scsi_virtqueue *svq = container_of(vq,  in vhost_scsi_get_cmd()
593 					struct vhost_scsi_virtqueue, vq);  in vhost_scsi_get_cmd()
627 	cmd->inflight = vhost_scsi_get_inflight(vq);  in vhost_scsi_get_cmd()
807 			   struct vhost_virtqueue *vq,  in vhost_scsi_send_bad_target()  argument
816 	resp = vq->iov[out].iov_base;  in vhost_scsi_send_bad_target()
819 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_send_bad_target()
825 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,  in vhost_scsi_get_desc()  argument
830 	vc->head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_get_desc()
831 				     ARRAY_SIZE(vq->iov), &vc->out, &vc->in,  in vhost_scsi_get_desc()
842 	if (vc->head == vq->num) {  in vhost_scsi_get_desc()
843 		if (unlikely(vhost_enable_notify(&vs->dev, vq))) {  in vhost_scsi_get_desc()
844 			vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_get_desc()
854 	vc->out_size = iov_length(vq->iov, vc->out);  in vhost_scsi_get_desc()
855 	vc->in_size = iov_length(&vq->iov[vc->out], vc->in);  in vhost_scsi_get_desc()
867 	iov_iter_init(&vc->out_iter, WRITE, vq->iov, vc->out, vc->out_size);  in vhost_scsi_get_desc()
875 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)  in vhost_scsi_chk_size()  argument
878 		vq_err(vq,  in vhost_scsi_chk_size()
883 		vq_err(vq,  in vhost_scsi_chk_size()
893 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,  in vhost_scsi_get_req()  argument
900 		vq_err(vq, "Faulted on copy_from_iter_full\n");  in vhost_scsi_get_req()
903 		vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);  in vhost_scsi_get_req()
907 		vs_tpg = vhost_vq_get_backend(vq);	/* validated at handler entry */  in vhost_scsi_get_req()
911 			vq_err(vq, "Target 0x%x does not exist\n", *vc->target);  in vhost_scsi_get_req()
928 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)  in vhost_scsi_handle_vq()  argument
941 	bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);  in vhost_scsi_handle_vq()
944 	mutex_lock(&vq->mutex);  in vhost_scsi_handle_vq()
949 	vs_tpg = vhost_vq_get_backend(vq);  in vhost_scsi_handle_vq()
956 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_handle_vq()
959 		ret = vhost_scsi_get_desc(vs, vq, &vc);  in vhost_scsi_handle_vq()
984 		ret = vhost_scsi_chk_size(vq, &vc);  in vhost_scsi_handle_vq()
988 		ret = vhost_scsi_get_req(vq, &vc, &tpg);  in vhost_scsi_handle_vq()
1019 			iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in,  in vhost_scsi_handle_vq()
1035 					vq_err(vq, "Received non zero pi_bytesout,"  in vhost_scsi_handle_vq()
1039 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);  in vhost_scsi_handle_vq()
1042 					vq_err(vq, "Received non zero pi_bytesin,"  in vhost_scsi_handle_vq()
1046 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);  in vhost_scsi_handle_vq()
1062 			tag = vhost64_to_cpu(vq, v_req_pi.tag);  in vhost_scsi_handle_vq()
1067 			tag = vhost64_to_cpu(vq, v_req.tag);  in vhost_scsi_handle_vq()
1080 			vq_err(vq, "Received SCSI CDB with command_size: %d that"  in vhost_scsi_handle_vq()
1085 		cmd = vhost_scsi_get_cmd(vq, tpg, cdb, tag, lun, task_attr,  in vhost_scsi_handle_vq()
1089 			vq_err(vq, "vhost_scsi_get_cmd failed %ld\n",  in vhost_scsi_handle_vq()
1094 		cmd->tvc_vq = vq;  in vhost_scsi_handle_vq()
1095 		cmd->tvc_resp_iov = vq->iov[vc.out];  in vhost_scsi_handle_vq()
1107 				vq_err(vq, "Failed to map iov to sgl\n");  in vhost_scsi_handle_vq()
1130 			vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);  in vhost_scsi_handle_vq()
1131 	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));  in vhost_scsi_handle_vq()
1133 	mutex_unlock(&vq->mutex);  in vhost_scsi_handle_vq()
1137 vhost_scsi_send_tmf_resp(struct vhost_scsi *vs, struct vhost_virtqueue *vq,  in vhost_scsi_send_tmf_resp()  argument
1153 		vhost_add_used_and_signal(&vs->dev, vq, vq_desc, 0);  in vhost_scsi_send_tmf_resp()
1169 	vhost_scsi_send_tmf_resp(tmf->vhost, &tmf->svq->vq, tmf->in_iovs,  in vhost_scsi_tmf_resp_work()
1176 		      struct vhost_virtqueue *vq,  in vhost_scsi_handle_tmf()  argument
1180 	struct vhost_scsi_virtqueue *svq = container_of(vq,  in vhost_scsi_handle_tmf()
1181 					struct vhost_scsi_virtqueue, vq);  in vhost_scsi_handle_tmf()
1184 	if (vhost32_to_cpu(vq, vtmf->subtype) !=  in vhost_scsi_handle_tmf()
1208 	tmf->resp_iov = vq->iov[vc->out];  in vhost_scsi_handle_tmf()
1211 	tmf->inflight = vhost_scsi_get_inflight(vq);  in vhost_scsi_handle_tmf()
1224 	vhost_scsi_send_tmf_resp(vs, vq, vc->in, vc->head, &vq->iov[vc->out],  in vhost_scsi_handle_tmf()
1230 			struct vhost_virtqueue *vq,  in vhost_scsi_send_an_resp()  argument
1241 	iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));  in vhost_scsi_send_an_resp()
1245 		vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);  in vhost_scsi_send_an_resp()
1251 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)  in vhost_scsi_ctl_handle_vq()  argument
1263 	mutex_lock(&vq->mutex);  in vhost_scsi_ctl_handle_vq()
1268 	if (!vhost_vq_get_backend(vq))  in vhost_scsi_ctl_handle_vq()
1273 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_ctl_handle_vq()
1276 		ret = vhost_scsi_get_desc(vs, vq, &vc);  in vhost_scsi_ctl_handle_vq()
1289 			vq_err(vq, "Faulted on copy_from_iter tmf type\n");  in vhost_scsi_ctl_handle_vq()
1299 		switch (vhost32_to_cpu(vq, v_req.type)) {  in vhost_scsi_ctl_handle_vq()
1316 			vq_err(vq, "Unknown control request %d", v_req.type);  in vhost_scsi_ctl_handle_vq()
1325 		ret = vhost_scsi_chk_size(vq, &vc);  in vhost_scsi_ctl_handle_vq()
1335 		ret = vhost_scsi_get_req(vq, &vc, &tpg);  in vhost_scsi_ctl_handle_vq()
1340 			vhost_scsi_handle_tmf(vs, tpg, vq, &v_req.tmf, &vc);  in vhost_scsi_ctl_handle_vq()
1342 			vhost_scsi_send_an_resp(vs, vq, &vc);  in vhost_scsi_ctl_handle_vq()
1353 			vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);  in vhost_scsi_ctl_handle_vq()
1354 	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));  in vhost_scsi_ctl_handle_vq()
1356 	mutex_unlock(&vq->mutex);  in vhost_scsi_ctl_handle_vq()
1361 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_ctl_handle_kick()  local
1363 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_ctl_handle_kick()
1366 	vhost_scsi_ctl_handle_vq(vs, vq);  in vhost_scsi_ctl_handle_kick()
1401 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_evt_handle_kick()  local
1403 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_evt_handle_kick()
1405 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1406 	if (!vhost_vq_get_backend(vq))  in vhost_scsi_evt_handle_kick()
1412 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1417 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_handle_kick()  local
1419 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_handle_kick()
1421 	vhost_scsi_handle_vq(vs, vq);  in vhost_scsi_handle_kick()
1448 static void vhost_scsi_destroy_vq_cmds(struct vhost_virtqueue *vq)  in vhost_scsi_destroy_vq_cmds()  argument
1450 	struct vhost_scsi_virtqueue *svq = container_of(vq,  in vhost_scsi_destroy_vq_cmds()
1451 					struct vhost_scsi_virtqueue, vq);  in vhost_scsi_destroy_vq_cmds()
1471 static int vhost_scsi_setup_vq_cmds(struct vhost_virtqueue *vq, int max_cmds)  in vhost_scsi_setup_vq_cmds()  argument
1473 	struct vhost_scsi_virtqueue *svq = container_of(vq,  in vhost_scsi_setup_vq_cmds()
1474 					struct vhost_scsi_virtqueue, vq);  in vhost_scsi_setup_vq_cmds()
1521 	vhost_scsi_destroy_vq_cmds(vq);  in vhost_scsi_setup_vq_cmds()
1540 	struct vhost_virtqueue *vq;  in vhost_scsi_set_endpoint()  local
1550 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_set_endpoint()
1609 			vq = &vs->vqs[i].vq;  in vhost_scsi_set_endpoint()
1610 			if (!vhost_vq_is_setup(vq))  in vhost_scsi_set_endpoint()
1613 			ret = vhost_scsi_setup_vq_cmds(vq, vq->num);  in vhost_scsi_set_endpoint()
1619 			vq = &vs->vqs[i].vq;  in vhost_scsi_set_endpoint()
1620 			mutex_lock(&vq->mutex);  in vhost_scsi_set_endpoint()
1621 			vhost_vq_set_backend(vq, vs_tpg);  in vhost_scsi_set_endpoint()
1622 			vhost_vq_init_access(vq);  in vhost_scsi_set_endpoint()
1623 			mutex_unlock(&vq->mutex);  in vhost_scsi_set_endpoint()
1641 		if (!vhost_vq_get_backend(&vs->vqs[i].vq))  in vhost_scsi_set_endpoint()
1642 			vhost_scsi_destroy_vq_cmds(&vs->vqs[i].vq);  in vhost_scsi_set_endpoint()
1666 	struct vhost_virtqueue *vq;  in vhost_scsi_clear_endpoint()  local
1675 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_clear_endpoint()
1721 			vq = &vs->vqs[i].vq;  in vhost_scsi_clear_endpoint()
1722 			mutex_lock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1723 			vhost_vq_set_backend(vq, NULL);  in vhost_scsi_clear_endpoint()
1724 			mutex_unlock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1730 			vq = &vs->vqs[i].vq;  in vhost_scsi_clear_endpoint()
1731 			vhost_scsi_destroy_vq_cmds(vq);  in vhost_scsi_clear_endpoint()
1756 	struct vhost_virtqueue *vq;  in vhost_scsi_set_features()  local
1770 		vq = &vs->vqs[i].vq;  in vhost_scsi_set_features()
1771 		mutex_lock(&vq->mutex);  in vhost_scsi_set_features()
1772 		vq->acked_features = features;  in vhost_scsi_set_features()
1773 		mutex_unlock(&vq->mutex);  in vhost_scsi_set_features()
1823 	vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;  in vhost_scsi_open()
1824 	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_open()
1825 	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;  in vhost_scsi_open()
1826 	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;  in vhost_scsi_open()
1828 		vqs[i] = &vs->vqs[i].vq;  in vhost_scsi_open()
1829 		vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;  in vhost_scsi_open()
1883 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_ioctl()  local
1907 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1909 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1912 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1914 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1985 	struct vhost_virtqueue *vq;  in vhost_scsi_do_plug()  local
1998 	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_plug()
1999 	mutex_lock(&vq->mutex);  in vhost_scsi_do_plug()
2000 	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))  in vhost_scsi_do_plug()
2003 	mutex_unlock(&vq->mutex);  in vhost_scsi_do_plug()