180 	struct vhost_virtqueue vq;  member
243 	struct vhost_virtqueue *vq;  in vhost_scsi_init_inflight()  local
247 		vq = &vs->vqs[i].vq;  in vhost_scsi_init_inflight()
249 		mutex_lock(&vq->mutex);  in vhost_scsi_init_inflight()
262 		mutex_unlock(&vq->mutex);  in vhost_scsi_init_inflight()
267 vhost_scsi_get_inflight(struct vhost_virtqueue *vq)  in vhost_scsi_get_inflight()  argument
272 	svq = container_of(vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_get_inflight()
410 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_allocate_evt()  local
420 		vq_err(vq, "Failed to allocate vhost_scsi_evt\n");  in vhost_scsi_allocate_evt()
425 	evt->event.event = cpu_to_vhost32(vq, event);  in vhost_scsi_allocate_evt()
426 	evt->event.reason = cpu_to_vhost32(vq, reason);  in vhost_scsi_allocate_evt()
449 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_evt_work()  local
455 	if (!vq->private_data) {  in vhost_scsi_do_evt_work()
461 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_do_evt_work()
462 	head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_do_evt_work()
463 			ARRAY_SIZE(vq->iov), &out, &in,  in vhost_scsi_do_evt_work()
469 	if (head == vq->num) {  in vhost_scsi_do_evt_work()
470 		if (vhost_enable_notify(&vs->dev, vq))  in vhost_scsi_do_evt_work()
476 	if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) {  in vhost_scsi_do_evt_work()
477 		vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n",  in vhost_scsi_do_evt_work()
478 				vq->iov[out].iov_len);  in vhost_scsi_do_evt_work()
484 		event->event |= cpu_to_vhost32(vq, VIRTIO_SCSI_T_EVENTS_MISSED);  in vhost_scsi_do_evt_work()
488 	eventp = vq->iov[out].iov_base;  in vhost_scsi_do_evt_work()
491 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_do_evt_work()
493 		vq_err(vq, "Faulted on vhost_scsi_send_event\n");  in vhost_scsi_do_evt_work()
500 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_evt_work()  local
504 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_work()
510 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_work()
528 	int ret, vq;  in vhost_scsi_complete_cmd_work()  local
553 			q = container_of(cmd->tvc_vq, struct vhost_scsi_virtqueue, vq);  in vhost_scsi_complete_cmd_work()
554 			vq = q - vs->vqs;  in vhost_scsi_complete_cmd_work()
555 			__set_bit(vq, signal);  in vhost_scsi_complete_cmd_work()
562 	vq = -1;  in vhost_scsi_complete_cmd_work()
563 	while ((vq = find_next_bit(signal, VHOST_SCSI_MAX_VQ, vq + 1))  in vhost_scsi_complete_cmd_work()
565 		vhost_signal(&vs->dev, &vs->vqs[vq].vq);  in vhost_scsi_complete_cmd_work()
569 vhost_scsi_get_tag(struct vhost_virtqueue *vq, struct vhost_scsi_tpg *tpg,  in vhost_scsi_get_tag()  argument
609 	cmd->inflight = vhost_scsi_get_inflight(vq);  in vhost_scsi_get_tag()
795 			   struct vhost_virtqueue *vq,  in vhost_scsi_send_bad_target()  argument
804 	resp = vq->iov[out].iov_base;  in vhost_scsi_send_bad_target()
807 		vhost_add_used_and_signal(&vs->dev, vq, head, 0);  in vhost_scsi_send_bad_target()
813 vhost_scsi_get_desc(struct vhost_scsi *vs, struct vhost_virtqueue *vq,  in vhost_scsi_get_desc()  argument
818 	vc->head = vhost_get_vq_desc(vq, vq->iov,  in vhost_scsi_get_desc()
819 				     ARRAY_SIZE(vq->iov), &vc->out, &vc->in,  in vhost_scsi_get_desc()
830 	if (vc->head == vq->num) {  in vhost_scsi_get_desc()
831 		if (unlikely(vhost_enable_notify(&vs->dev, vq))) {  in vhost_scsi_get_desc()
832 			vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_get_desc()
842 	vc->out_size = iov_length(vq->iov, vc->out);  in vhost_scsi_get_desc()
843 	vc->in_size = iov_length(&vq->iov[vc->out], vc->in);  in vhost_scsi_get_desc()
855 	iov_iter_init(&vc->out_iter, WRITE, vq->iov, vc->out, vc->out_size);  in vhost_scsi_get_desc()
863 vhost_scsi_chk_size(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc)  in vhost_scsi_chk_size()  argument
866 		vq_err(vq,  in vhost_scsi_chk_size()
871 		vq_err(vq,  in vhost_scsi_chk_size()
881 vhost_scsi_get_req(struct vhost_virtqueue *vq, struct vhost_scsi_ctx *vc,  in vhost_scsi_get_req()  argument
888 		vq_err(vq, "Faulted on copy_from_iter_full\n");  in vhost_scsi_get_req()
891 		vq_err(vq, "Illegal virtio-scsi lun: %u\n", *vc->lunp);  in vhost_scsi_get_req()
895 		vs_tpg = vq->private_data;	/* validated at handler entry */  in vhost_scsi_get_req()
899 			vq_err(vq, "Target 0x%x does not exist\n", *vc->target);  in vhost_scsi_get_req()
911 vhost_scsi_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)  in vhost_scsi_handle_vq()  argument
924 	bool t10_pi = vhost_has_feature(vq, VIRTIO_SCSI_F_T10_PI);  in vhost_scsi_handle_vq()
927 	mutex_lock(&vq->mutex);  in vhost_scsi_handle_vq()
932 	vs_tpg = vq->private_data;  in vhost_scsi_handle_vq()
939 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_handle_vq()
942 		ret = vhost_scsi_get_desc(vs, vq, &vc);  in vhost_scsi_handle_vq()
967 		ret = vhost_scsi_chk_size(vq, &vc);  in vhost_scsi_handle_vq()
971 		ret = vhost_scsi_get_req(vq, &vc, &tpg);  in vhost_scsi_handle_vq()
1002 			iov_iter_init(&in_iter, READ, &vq->iov[vc.out], vc.in,  in vhost_scsi_handle_vq()
1018 					vq_err(vq, "Received non zero pi_bytesout,"  in vhost_scsi_handle_vq()
1022 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesout);  in vhost_scsi_handle_vq()
1025 					vq_err(vq, "Received non zero pi_bytesin,"  in vhost_scsi_handle_vq()
1029 				prot_bytes = vhost32_to_cpu(vq, v_req_pi.pi_bytesin);  in vhost_scsi_handle_vq()
1045 			tag = vhost64_to_cpu(vq, v_req_pi.tag);  in vhost_scsi_handle_vq()
1050 			tag = vhost64_to_cpu(vq, v_req.tag);  in vhost_scsi_handle_vq()
1063 			vq_err(vq, "Received SCSI CDB with command_size: %d that"  in vhost_scsi_handle_vq()
1068 		cmd = vhost_scsi_get_tag(vq, tpg, cdb, tag, lun, task_attr,  in vhost_scsi_handle_vq()
1072 			vq_err(vq, "vhost_scsi_get_tag failed %ld\n",  in vhost_scsi_handle_vq()
1077 		cmd->tvc_vq = vq;  in vhost_scsi_handle_vq()
1078 		cmd->tvc_resp_iov = vq->iov[vc.out];  in vhost_scsi_handle_vq()
1090 				vq_err(vq, "Failed to map iov to sgl\n");  in vhost_scsi_handle_vq()
1120 			vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);  in vhost_scsi_handle_vq()
1121 	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));  in vhost_scsi_handle_vq()
1123 	mutex_unlock(&vq->mutex);  in vhost_scsi_handle_vq()
1128 			   struct vhost_virtqueue *vq,  in vhost_scsi_send_tmf_reject()  argument
1139 	iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));  in vhost_scsi_send_tmf_reject()
1143 		vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);  in vhost_scsi_send_tmf_reject()
1150 			struct vhost_virtqueue *vq,  in vhost_scsi_send_an_resp()  argument
1161 	iov_iter_init(&iov_iter, READ, &vq->iov[vc->out], vc->in, sizeof(rsp));  in vhost_scsi_send_an_resp()
1165 		vhost_add_used_and_signal(&vs->dev, vq, vc->head, 0);  in vhost_scsi_send_an_resp()
1171 vhost_scsi_ctl_handle_vq(struct vhost_scsi *vs, struct vhost_virtqueue *vq)  in vhost_scsi_ctl_handle_vq()  argument
1182 	mutex_lock(&vq->mutex);  in vhost_scsi_ctl_handle_vq()
1187 	if (!vq->private_data)  in vhost_scsi_ctl_handle_vq()
1192 	vhost_disable_notify(&vs->dev, vq);  in vhost_scsi_ctl_handle_vq()
1195 		ret = vhost_scsi_get_desc(vs, vq, &vc);  in vhost_scsi_ctl_handle_vq()
1208 			vq_err(vq, "Faulted on copy_from_iter tmf type\n");  in vhost_scsi_ctl_handle_vq()
1235 			vq_err(vq, "Unknown control request %d", v_req.type);  in vhost_scsi_ctl_handle_vq()
1244 		ret = vhost_scsi_chk_size(vq, &vc);  in vhost_scsi_ctl_handle_vq()
1254 		ret = vhost_scsi_get_req(vq, &vc, NULL);  in vhost_scsi_ctl_handle_vq()
1259 			vhost_scsi_send_tmf_reject(vs, vq, &vc);  in vhost_scsi_ctl_handle_vq()
1261 			vhost_scsi_send_an_resp(vs, vq, &vc);  in vhost_scsi_ctl_handle_vq()
1272 			vhost_scsi_send_bad_target(vs, vq, vc.head, vc.out);  in vhost_scsi_ctl_handle_vq()
1273 	} while (likely(!vhost_exceeds_weight(vq, ++c, 0)));  in vhost_scsi_ctl_handle_vq()
1275 	mutex_unlock(&vq->mutex);  in vhost_scsi_ctl_handle_vq()
1280 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_ctl_handle_kick()  local
1282 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_ctl_handle_kick()
1285 	vhost_scsi_ctl_handle_vq(vs, vq);  in vhost_scsi_ctl_handle_kick()
1320 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_evt_handle_kick()  local
1322 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_evt_handle_kick()
1324 	mutex_lock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1325 	if (!vq->private_data)  in vhost_scsi_evt_handle_kick()
1331 	mutex_unlock(&vq->mutex);  in vhost_scsi_evt_handle_kick()
1336 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_scsi_handle_kick()  local
1338 	struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev);  in vhost_scsi_handle_kick()
1340 	vhost_scsi_handle_vq(vs, vq);  in vhost_scsi_handle_kick()
1345 	vhost_poll_flush(&vs->vqs[index].vq.poll);  in vhost_scsi_flush_vq()
1391 	struct vhost_virtqueue *vq;  in vhost_scsi_set_endpoint()  local
1401 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_set_endpoint()
1461 			vq = &vs->vqs[i].vq;  in vhost_scsi_set_endpoint()
1462 			mutex_lock(&vq->mutex);  in vhost_scsi_set_endpoint()
1463 			vq->private_data = vs_tpg;  in vhost_scsi_set_endpoint()
1464 			vhost_vq_init_access(vq);  in vhost_scsi_set_endpoint()
1465 			mutex_unlock(&vq->mutex);  in vhost_scsi_set_endpoint()
1493 	struct vhost_virtqueue *vq;  in vhost_scsi_clear_endpoint()  local
1502 		if (!vhost_vq_access_ok(&vs->vqs[index].vq)) {  in vhost_scsi_clear_endpoint()
1548 			vq = &vs->vqs[i].vq;  in vhost_scsi_clear_endpoint()
1549 			mutex_lock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1550 			vq->private_data = NULL;  in vhost_scsi_clear_endpoint()
1551 			mutex_unlock(&vq->mutex);  in vhost_scsi_clear_endpoint()
1576 	struct vhost_virtqueue *vq;  in vhost_scsi_set_features()  local
1590 		vq = &vs->vqs[i].vq;  in vhost_scsi_set_features()
1591 		mutex_lock(&vq->mutex);  in vhost_scsi_set_features()
1592 		vq->acked_features = features;  in vhost_scsi_set_features()
1593 		mutex_unlock(&vq->mutex);  in vhost_scsi_set_features()
1622 	vqs[VHOST_SCSI_VQ_CTL] = &vs->vqs[VHOST_SCSI_VQ_CTL].vq;  in vhost_scsi_open()
1623 	vqs[VHOST_SCSI_VQ_EVT] = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_open()
1624 	vs->vqs[VHOST_SCSI_VQ_CTL].vq.handle_kick = vhost_scsi_ctl_handle_kick;  in vhost_scsi_open()
1625 	vs->vqs[VHOST_SCSI_VQ_EVT].vq.handle_kick = vhost_scsi_evt_handle_kick;  in vhost_scsi_open()
1627 		vqs[i] = &vs->vqs[i].vq;  in vhost_scsi_open()
1628 		vs->vqs[i].vq.handle_kick = vhost_scsi_handle_kick;  in vhost_scsi_open()
1675 	struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_ioctl()  local
1699 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1701 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1704 		mutex_lock(&vq->mutex);  in vhost_scsi_ioctl()
1706 		mutex_unlock(&vq->mutex);  in vhost_scsi_ioctl()
1787 	struct vhost_virtqueue *vq;  in vhost_scsi_do_plug()  local
1800 	vq = &vs->vqs[VHOST_SCSI_VQ_EVT].vq;  in vhost_scsi_do_plug()
1801 	mutex_lock(&vq->mutex);  in vhost_scsi_do_plug()
1802 	if (vhost_has_feature(vq, VIRTIO_SCSI_F_HOTPLUG))  in vhost_scsi_do_plug()
1805 	mutex_unlock(&vq->mutex);  in vhost_scsi_do_plug()