85 			    struct vhost_virtqueue *vq)  in vhost_transport_do_send_pkt()  argument
92 	mutex_lock(&vq->mutex);  in vhost_transport_do_send_pkt()
94 	if (!vq->private_data)  in vhost_transport_do_send_pkt()
98 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
111 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
120 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_transport_do_send_pkt()
129 		if (head == vq->num) {  in vhost_transport_do_send_pkt()
137 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
138 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
146 			vq_err(vq, "Expected 0 output buffers, got %u\n", out);  in vhost_transport_do_send_pkt()
150 		iov_len = iov_length(&vq->iov[out], in);  in vhost_transport_do_send_pkt()
153 			vq_err(vq, "Buffer len [%zu] too small\n", iov_len);  in vhost_transport_do_send_pkt()
157 		iov_iter_init(&iov_iter, READ, &vq->iov[out], in, iov_len);  in vhost_transport_do_send_pkt()
172 			vq_err(vq, "Faulted on copying pkt hdr\n");  in vhost_transport_do_send_pkt()
180 			vq_err(vq, "Faulted on copying pkt buf\n");  in vhost_transport_do_send_pkt()
184 		vhost_add_used(vq, head, sizeof(pkt->hdr) + payload_len);  in vhost_transport_do_send_pkt()
217 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_transport_do_send_pkt()
219 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
222 	mutex_unlock(&vq->mutex);  in vhost_transport_do_send_pkt()
230 	struct vhost_virtqueue *vq;  in vhost_transport_send_pkt_work()  local
234 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
236 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
315 vhost_vsock_alloc_pkt(struct vhost_virtqueue *vq,  in vhost_vsock_alloc_pkt()  argument
324 		vq_err(vq, "Expected 0 input buffers, got %u\n", in);  in vhost_vsock_alloc_pkt()
332 	len = iov_length(vq->iov, out);  in vhost_vsock_alloc_pkt()
333 	iov_iter_init(&iov_iter, WRITE, vq->iov, out, len);  in vhost_vsock_alloc_pkt()
337 		vq_err(vq, "Expected %zu bytes for pkt->hdr, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
366 		vq_err(vq, "Expected %u byte payload, got %zu bytes\n",  in vhost_vsock_alloc_pkt()
378 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()  local
384 	return val < vq->num;  in vhost_vsock_more_replies()
389 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_tx_kick()  local
391 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()
398 	mutex_lock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
400 	if (!vq->private_data)  in vhost_vsock_handle_tx_kick()
403 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
415 		head = vhost_get_vq_desc(vq, vq->iov, ARRAY_SIZE(vq->iov),  in vhost_vsock_handle_tx_kick()
420 		if (head == vq->num) {  in vhost_vsock_handle_tx_kick()
421 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
422 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
428 		pkt = vhost_vsock_alloc_pkt(vq, out, in);  in vhost_vsock_handle_tx_kick()
430 			vq_err(vq, "Faulted on pkt\n");  in vhost_vsock_handle_tx_kick()
446 		vhost_add_used(vq, head, len);  in vhost_vsock_handle_tx_kick()
449 	} while(likely(!vhost_exceeds_weight(vq, ++pkts, total_len)));  in vhost_vsock_handle_tx_kick()
453 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
456 	mutex_unlock(&vq->mutex);  in vhost_vsock_handle_tx_kick()
461 	struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue,  in vhost_vsock_handle_rx_kick()  local
463 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()
466 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
471 	struct vhost_virtqueue *vq;  in vhost_vsock_start()  local
482 		vq = &vsock->vqs[i];  in vhost_vsock_start()
484 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
486 		if (!vhost_vq_access_ok(vq)) {  in vhost_vsock_start()
491 		if (!vq->private_data) {  in vhost_vsock_start()
492 			vq->private_data = vsock;  in vhost_vsock_start()
493 			ret = vhost_vq_init_access(vq);  in vhost_vsock_start()
498 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
505 	vq->private_data = NULL;  in vhost_vsock_start()
506 	mutex_unlock(&vq->mutex);  in vhost_vsock_start()
509 		vq = &vsock->vqs[i];  in vhost_vsock_start()
511 		mutex_lock(&vq->mutex);  in vhost_vsock_start()
512 		vq->private_data = NULL;  in vhost_vsock_start()
513 		mutex_unlock(&vq->mutex);  in vhost_vsock_start()
532 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()  local
534 		mutex_lock(&vq->mutex);  in vhost_vsock_stop()
535 		vq->private_data = NULL;  in vhost_vsock_stop()
536 		mutex_unlock(&vq->mutex);  in vhost_vsock_stop()
698 	struct vhost_virtqueue *vq;  in vhost_vsock_set_features()  local
712 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
713 		mutex_lock(&vq->mutex);  in vhost_vsock_set_features()
714 		vq->acked_features = features;  in vhost_vsock_set_features()
715 		mutex_unlock(&vq->mutex);  in vhost_vsock_set_features()