85 			    struct vhost_virtqueue *vq)  in vhost_transport_do_send_pkt()  argument
91 	mutex_lock(&vq->mutex);  in vhost_transport_do_send_pkt()
93 	if (!vq->private_data)  in vhost_transport_do_send_pkt()
97 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
110 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
119 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_transport_do_send_pkt()
128 		if (head == vq->num) {  in vhost_transport_do_send_pkt()
136 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
137 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
145 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);  in vhost_transport_do_send_pkt()
149 		len = iov_length(&vq->iov[out], in);  in vhost_transport_do_send_pkt()
150 		iov_iter_init(&iov_iter, READ, &vq->iov[out], in, len);  in vhost_transport_do_send_pkt()
155 			vq_err(vq, "Faulted on copying pkt hdr\n");  in vhost_transport_do_send_pkt()
162 			vq_err(vq, "Faulted on copying pkt buf\n");  in vhost_transport_do_send_pkt()
166 		vhost_add_used(vq, head, sizeof(pkt->hdr) + pkt->len);  in vhost_transport_do_send_pkt()
187 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
190 	mutex_unlock(&vq->mutex);  in vhost_transport_do_send_pkt()
198 	struct vhost_virtqueue *vq;  in vhost_transport_send_pkt_work()  local
202 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
204 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
272 vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,  in vhost_vsock_alloc_pkt()  argument
281 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);  in vhost_vsock_alloc_pkt()
289 	len = iov_length(vq->iov, out);  in vhost_vsock_alloc_pkt()
290 	iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);  in vhost_vsock_alloc_pkt()
294 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
321 		vq_err(vq, "Expected %u byte payload, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
333 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()  local
339 	return val < vq->num;  in vhost_vsock_more_replies()
344 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_tx_kick()  local
346 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()
353 	mutex_lock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
355 	if (!vq->private_data)  in vhost_vsock_handle_tx_kick()
358 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
370 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_vsock_handle_tx_kick()
375 		if (head == vq->num) {  in vhost_vsock_handle_tx_kick()
376 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
377 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
383 		pkt = vhost_vsock_alloc_pkt(vq, out, in);  in vhost_vsock_handle_tx_kick()
385 			vq_err(vq, "Faulted on pkt\n");  in vhost_vsock_handle_tx_kick()
400 		vhost_add_used(vq, head, sizeof(pkt->hdr) + len);  in vhost_vsock_handle_tx_kick()
406 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
409 	mutex_unlock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
414 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_rx_kick()  local
416 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()
419 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
424 	struct vhost_virtqueue *vq;  in vhost_vsock_start()  local
435 		vq = &vsock->vqs[i];  in vhost_vsock_start()
437 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
439 		if (!vhost_vq_access_ok(vq)) {  in vhost_vsock_start()
444 		if (!vq->private_data) {  in vhost_vsock_start()
445 			vq->private_data = vsock;  in vhost_vsock_start()
446 			ret = vhost_vq_init_access(vq);  in vhost_vsock_start()
451 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
458 	vq->private_data = NULL;  in vhost_vsock_start()
459 	mutex_unlock(&vq->mutex);  in vhost_vsock_start()
462 		vq = &vsock->vqs[i];  in vhost_vsock_start()
464 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
465 		vq->private_data = NULL;  in vhost_vsock_start()
466 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
485 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()  local
487 		mutex_lock(&vq->mutex);  in vhost_vsock_stop()
488 		vq->private_data = NULL;  in vhost_vsock_stop()
489 		mutex_unlock(&vq->mutex);  in vhost_vsock_stop()
636 	struct vhost_virtqueue *vq;  in vhost_vsock_set_features()  local
650 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
651 		mutex_lock(&vq->mutex);  in vhost_vsock_set_features()
652 		vq->acked_features = features;  in vhost_vsock_set_features()
653 		mutex_unlock(&vq->mutex);  in vhost_vsock_set_features()