76 	struct virtio_vsock *vsock = virtio_vsock_get();  in virtio_transport_get_local_cid()  local
78 	return vsock->guest_cid;  in virtio_transport_get_local_cid()
83 	struct virtio_vsock *vsock =  in virtio_transport_loopback_work()  local
87 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_transport_loopback_work()
88 	list_splice_init(&vsock->loopback_list, &pkts);  in virtio_transport_loopback_work()
89 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_transport_loopback_work()
91 	mutex_lock(&vsock->rx_lock);  in virtio_transport_loopback_work()
100 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_loopback_work()
103 static int virtio_transport_send_pkt_loopback(struct virtio_vsock *vsock,  in virtio_transport_send_pkt_loopback()  argument
108 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_transport_send_pkt_loopback()
109 	list_add_tail(&pkt->list, &vsock->loopback_list);  in virtio_transport_send_pkt_loopback()
110 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_transport_send_pkt_loopback()
112 	queue_work(virtio_vsock_workqueue, &vsock->loopback_work);  in virtio_transport_send_pkt_loopback()
120 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
126 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
128 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
136 		spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
137 		if (list_empty(&vsock->send_pkt_list)) {  in virtio_transport_send_pkt_work()
138 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
142 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_transport_send_pkt_work()
145 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
163 			spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
164 			list_add(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt_work()
165 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
170 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
173 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
186 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
189 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
195 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
198 	vsock = virtio_vsock_get();  in virtio_transport_send_pkt()
199 	if (!vsock) {  in virtio_transport_send_pkt()
204 	if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid)  in virtio_transport_send_pkt()
205 		return virtio_transport_send_pkt_loopback(vsock, pkt);  in virtio_transport_send_pkt()
208 		atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
210 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
211 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt()
212 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
214 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
221 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
226 	vsock = virtio_vsock_get();  in virtio_transport_cancel_pkt()
227 	if (!vsock) {  in virtio_transport_cancel_pkt()
231 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
232 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in virtio_transport_cancel_pkt()
237 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
247 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
250 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
253 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
259 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
267 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
292 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
294 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
295 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
301 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
306 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
307 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
318 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
321 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
325 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
327 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
331 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
338 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
342 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
344 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
352 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
365 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
381 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
382 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
383 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
387 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
393 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
401 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
405 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
406 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
408 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
411 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
423 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
425 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
430 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
434 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
439 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
447 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
451 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
453 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
462 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
464 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
468 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
470 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
475 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
477 	if (!vsock)  in virtio_vsock_event_done()
479 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
484 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
486 	if (!vsock)  in virtio_vsock_tx_done()
488 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
493 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
495 	if (!vsock)  in virtio_vsock_rx_done()
497 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
558 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
571 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
572 	if (!vsock) {  in virtio_vsock_probe()
577 	vsock->vdev = vdev;  in virtio_vsock_probe()
579 	ret = virtio_find_vqs(vsock->vdev, VSOCK_VQ_MAX,  in virtio_vsock_probe()
580 			      vsock->vqs, callbacks, names,  in virtio_vsock_probe()
585 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_probe()
591 	vsock->rx_buf_nr = 0;  in virtio_vsock_probe()
592 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_probe()
593 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_probe()
595 	vdev->priv = vsock;  in virtio_vsock_probe()
596 	the_virtio_vsock = vsock;  in virtio_vsock_probe()
597 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
598 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
599 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
600 	spin_lock_init(&vsock->send_pkt_list_lock);  in virtio_vsock_probe()
601 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in virtio_vsock_probe()
602 	spin_lock_init(&vsock->loopback_list_lock);  in virtio_vsock_probe()
603 	INIT_LIST_HEAD(&vsock->loopback_list);  in virtio_vsock_probe()
604 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
605 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
606 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
607 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
608 	INIT_WORK(&vsock->loopback_work, virtio_transport_loopback_work);  in virtio_vsock_probe()
610 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_probe()
611 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_probe()
612 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_probe()
614 	mutex_lock(&vsock->event_lock);  in virtio_vsock_probe()
615 	virtio_vsock_event_fill(vsock);  in virtio_vsock_probe()
616 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_probe()
622 	vsock->vdev->config->del_vqs(vsock->vdev);  in virtio_vsock_probe()
624 	kfree(vsock);  in virtio_vsock_probe()
631 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
634 	flush_work(&vsock->loopback_work);  in virtio_vsock_remove()
635 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
636 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
637 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
638 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
642 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
643 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_remove()
645 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
647 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
648 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_remove()
650 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
652 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
653 	while (!list_empty(&vsock->send_pkt_list)) {  in virtio_vsock_remove()
654 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_vsock_remove()
659 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
661 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_vsock_remove()
662 	while (!list_empty(&vsock->loopback_list)) {  in virtio_vsock_remove()
663 		pkt = list_first_entry(&vsock->loopback_list,  in virtio_vsock_remove()
668 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_vsock_remove()
677 	kfree(vsock);  in virtio_vsock_remove()