73 	struct virtio_vsock *vsock;  in virtio_transport_get_local_cid()  local
77 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_get_local_cid()
78 	if (!vsock) {  in virtio_transport_get_local_cid()
83 	ret = vsock->guest_cid;  in virtio_transport_get_local_cid()
91 	struct virtio_vsock *vsock =  in virtio_transport_loopback_work()  local
95 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_transport_loopback_work()
96 	list_splice_init(&vsock->loopback_list, &pkts);  in virtio_transport_loopback_work()
97 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_transport_loopback_work()
99 	mutex_lock(&vsock->rx_lock);  in virtio_transport_loopback_work()
101 	if (!vsock->rx_run)  in virtio_transport_loopback_work()
113 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_loopback_work()
116 static int virtio_transport_send_pkt_loopback(struct virtio_vsock *vsock,  in virtio_transport_send_pkt_loopback()  argument
121 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_transport_send_pkt_loopback()
122 	list_add_tail(&pkt->list, &vsock->loopback_list);  in virtio_transport_send_pkt_loopback()
123 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_transport_send_pkt_loopback()
125 	queue_work(virtio_vsock_workqueue, &vsock->loopback_work);  in virtio_transport_send_pkt_loopback()
133 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
139 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
141 	if (!vsock->tx_run)  in virtio_transport_send_pkt_work()
144 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
152 		spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
153 		if (list_empty(&vsock->send_pkt_list)) {  in virtio_transport_send_pkt_work()
154 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
158 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_transport_send_pkt_work()
161 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
179 			spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
180 			list_add(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt_work()
181 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
186 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
189 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
203 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
206 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
212 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
216 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_send_pkt()
217 	if (!vsock) {  in virtio_transport_send_pkt()
223 	if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid) {  in virtio_transport_send_pkt()
224 		len = virtio_transport_send_pkt_loopback(vsock, pkt);  in virtio_transport_send_pkt()
229 		atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
231 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
232 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt()
233 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
235 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
245 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
251 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_cancel_pkt()
252 	if (!vsock) {  in virtio_transport_cancel_pkt()
257 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
258 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in virtio_transport_cancel_pkt()
263 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
273 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
276 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
279 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
289 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
297 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
323 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
325 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
326 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
332 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
337 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
338 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
340 	if (!vsock->tx_run)  in virtio_transport_tx_work()
355 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
358 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
362 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
364 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
368 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
375 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
379 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
381 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
383 	if (!vsock->rx_run)  in virtio_transport_rx_work()
392 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
405 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
421 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
422 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
423 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
427 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
433 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
441 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
445 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
446 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
448 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
451 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
463 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
465 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
470 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
474 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
479 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
487 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
491 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
493 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
495 	if (!vsock->event_run)  in virtio_transport_event_work()
505 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
507 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
511 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
513 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
518 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
520 	if (!vsock)  in virtio_vsock_event_done()
522 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
527 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
529 	if (!vsock)  in virtio_vsock_tx_done()
531 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
536 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
538 	if (!vsock)  in virtio_vsock_rx_done()
540 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
601 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
615 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
616 	if (!vsock) {  in virtio_vsock_probe()
621 	vsock->vdev = vdev;  in virtio_vsock_probe()
623 	ret = virtio_find_vqs(vsock->vdev, VSOCK_VQ_MAX,  in virtio_vsock_probe()
624 			      vsock->vqs, callbacks, names,  in virtio_vsock_probe()
629 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_probe()
631 	vsock->rx_buf_nr = 0;  in virtio_vsock_probe()
632 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_probe()
633 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_probe()
635 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
636 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
637 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
638 	spin_lock_init(&vsock->send_pkt_list_lock);  in virtio_vsock_probe()
639 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in virtio_vsock_probe()
640 	spin_lock_init(&vsock->loopback_list_lock);  in virtio_vsock_probe()
641 	INIT_LIST_HEAD(&vsock->loopback_list);  in virtio_vsock_probe()
642 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
643 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
644 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
645 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
646 	INIT_WORK(&vsock->loopback_work, virtio_transport_loopback_work);  in virtio_vsock_probe()
648 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_probe()
649 	vsock->tx_run = true;  in virtio_vsock_probe()
650 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_probe()
652 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_probe()
653 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_probe()
654 	vsock->rx_run = true;  in virtio_vsock_probe()
655 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_probe()
657 	mutex_lock(&vsock->event_lock);  in virtio_vsock_probe()
658 	virtio_vsock_event_fill(vsock);  in virtio_vsock_probe()
659 	vsock->event_run = true;  in virtio_vsock_probe()
660 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_probe()
662 	vdev->priv = vsock;  in virtio_vsock_probe()
663 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_probe()
669 	kfree(vsock);  in virtio_vsock_probe()
676 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
691 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
692 	vsock->rx_run = false;  in virtio_vsock_remove()
693 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
695 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
696 	vsock->tx_run = false;  in virtio_vsock_remove()
697 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
699 	mutex_lock(&vsock->event_lock);  in virtio_vsock_remove()
700 	vsock->event_run = false;  in virtio_vsock_remove()
701 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_remove()
708 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
709 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_remove()
711 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
713 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
714 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_remove()
716 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
718 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
719 	while (!list_empty(&vsock->send_pkt_list)) {  in virtio_vsock_remove()
720 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_vsock_remove()
725 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
727 	spin_lock_bh(&vsock->loopback_list_lock);  in virtio_vsock_remove()
728 	while (!list_empty(&vsock->loopback_list)) {  in virtio_vsock_remove()
729 		pkt = list_first_entry(&vsock->loopback_list,  in virtio_vsock_remove()
734 	spin_unlock_bh(&vsock->loopback_list_lock);  in virtio_vsock_remove()
742 	flush_work(&vsock->loopback_work);  in virtio_vsock_remove()
743 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
744 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
745 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
746 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
750 	kfree(vsock);  in virtio_vsock_remove()