71 	struct virtio_vsock *vsock;  in virtio_transport_get_local_cid()  local
75 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_get_local_cid()
76 	if (!vsock) {  in virtio_transport_get_local_cid()
81 	ret = vsock->guest_cid;  in virtio_transport_get_local_cid()
90 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
96 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
98 	if (!vsock->tx_run)  in virtio_transport_send_pkt_work()
101 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
109 		spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
110 		if (list_empty(&vsock->send_pkt_list)) {  in virtio_transport_send_pkt_work()
111 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
115 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_transport_send_pkt_work()
118 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
136 			spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
137 			list_add(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt_work()
138 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
143 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
146 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
160 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
163 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
169 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
173 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_send_pkt()
174 	if (!vsock) {  in virtio_transport_send_pkt()
180 	if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid) {  in virtio_transport_send_pkt()
187 		atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
189 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
190 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt()
191 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
193 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
203 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
209 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_cancel_pkt()
210 	if (!vsock) {  in virtio_transport_cancel_pkt()
215 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
216 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in virtio_transport_cancel_pkt()
221 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
231 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
234 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
237 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
247 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
255 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
281 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
283 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
284 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
290 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
295 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
296 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
298 	if (!vsock->tx_run)  in virtio_transport_tx_work()
313 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
316 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
320 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
322 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
326 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
332 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
338 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
346 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
350 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
351 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
353 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
356 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
371 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
373 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
378 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
382 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
387 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
396 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
400 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
402 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
404 	if (!vsock->event_run)  in virtio_transport_event_work()
414 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
416 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
420 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
422 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
427 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
429 	if (!vsock)  in virtio_vsock_event_done()
431 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
436 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
438 	if (!vsock)  in virtio_vsock_tx_done()
440 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
445 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
447 	if (!vsock)  in virtio_vsock_rx_done()
449 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
503 	struct virtio_vsock *vsock;  in virtio_transport_seqpacket_allow()  local
508 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_seqpacket_allow()
509 	if (vsock)  in virtio_transport_seqpacket_allow()
510 		seqpacket_allow = vsock->seqpacket_allow;  in virtio_transport_seqpacket_allow()
518 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
522 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
524 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
526 	if (!vsock->rx_run)  in virtio_transport_rx_work()
535 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
548 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
564 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
565 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
566 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
569 static int virtio_vsock_vqs_init(struct virtio_vsock *vsock)  in virtio_vsock_vqs_init()  argument
571 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_vqs_init()
584 	ret = virtio_find_vqs(vdev, VSOCK_VQ_MAX, vsock->vqs, callbacks, names,  in virtio_vsock_vqs_init()
589 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_vqs_init()
593 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_init()
594 	vsock->tx_run = true;  in virtio_vsock_vqs_init()
595 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_init()
597 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_init()
598 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_vqs_init()
599 	vsock->rx_run = true;  in virtio_vsock_vqs_init()
600 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_init()
602 	mutex_lock(&vsock->event_lock);  in virtio_vsock_vqs_init()
603 	virtio_vsock_event_fill(vsock);  in virtio_vsock_vqs_init()
604 	vsock->event_run = true;  in virtio_vsock_vqs_init()
605 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_vqs_init()
610 static void virtio_vsock_vqs_del(struct virtio_vsock *vsock)  in virtio_vsock_vqs_del()  argument
612 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_vqs_del()
622 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
623 	vsock->rx_run = false;  in virtio_vsock_vqs_del()
624 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
626 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
627 	vsock->tx_run = false;  in virtio_vsock_vqs_del()
628 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
630 	mutex_lock(&vsock->event_lock);  in virtio_vsock_vqs_del()
631 	vsock->event_run = false;  in virtio_vsock_vqs_del()
632 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_vqs_del()
639 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
640 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_vqs_del()
642 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_vqs_del()
644 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
645 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_vqs_del()
647 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_vqs_del()
649 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_vqs_del()
650 	while (!list_empty(&vsock->send_pkt_list)) {  in virtio_vsock_vqs_del()
651 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_vsock_vqs_del()
656 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_vqs_del()
664 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
678 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
679 	if (!vsock) {  in virtio_vsock_probe()
684 	vsock->vdev = vdev;  in virtio_vsock_probe()
686 	vsock->rx_buf_nr = 0;  in virtio_vsock_probe()
687 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_probe()
688 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_probe()
690 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
691 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
692 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
693 	spin_lock_init(&vsock->send_pkt_list_lock);  in virtio_vsock_probe()
694 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in virtio_vsock_probe()
695 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
696 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
697 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
698 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
701 		vsock->seqpacket_allow = true;  in virtio_vsock_probe()
703 	vdev->priv = vsock;  in virtio_vsock_probe()
705 	ret = virtio_vsock_vqs_init(vsock);  in virtio_vsock_probe()
709 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_probe()
716 	kfree(vsock);  in virtio_vsock_probe()
723 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
731 	virtio_vsock_vqs_del(vsock);  in virtio_vsock_remove()
736 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
737 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
738 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
739 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
743 	kfree(vsock);  in virtio_vsock_remove()
749 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_freeze()  local
756 	virtio_vsock_vqs_del(vsock);  in virtio_vsock_freeze()
765 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_restore()  local
777 	ret = virtio_vsock_vqs_init(vsock);  in virtio_vsock_restore()
781 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_restore()