69 	struct virtio_vsock *vsock;  in virtio_transport_get_local_cid()  local
73 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_get_local_cid()
74 	if (!vsock) {  in virtio_transport_get_local_cid()
79 	ret = vsock->guest_cid;  in virtio_transport_get_local_cid()
88 	struct virtio_vsock *vsock =  in virtio_transport_send_pkt_work()  local
94 	mutex_lock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
96 	if (!vsock->tx_run)  in virtio_transport_send_pkt_work()
99 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_send_pkt_work()
107 		spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
108 		if (list_empty(&vsock->send_pkt_list)) {  in virtio_transport_send_pkt_work()
109 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
113 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_transport_send_pkt_work()
116 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
134 			spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
135 			list_add(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt_work()
136 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt_work()
141 			struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_send_pkt_work()
144 			val = atomic_dec_return(&vsock->queued_replies);  in virtio_transport_send_pkt_work()
158 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_send_pkt_work()
161 		queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_send_pkt_work()
167 	struct virtio_vsock *vsock;  in virtio_transport_send_pkt()  local
171 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_send_pkt()
172 	if (!vsock) {  in virtio_transport_send_pkt()
178 	if (le64_to_cpu(pkt->hdr.dst_cid) == vsock->guest_cid) {  in virtio_transport_send_pkt()
185 		atomic_inc(&vsock->queued_replies);  in virtio_transport_send_pkt()
187 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
188 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in virtio_transport_send_pkt()
189 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_send_pkt()
191 	queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_send_pkt()
201 	struct virtio_vsock *vsock;  in virtio_transport_cancel_pkt()  local
207 	vsock = rcu_dereference(the_virtio_vsock);  in virtio_transport_cancel_pkt()
208 	if (!vsock) {  in virtio_transport_cancel_pkt()
213 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
214 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in virtio_transport_cancel_pkt()
219 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_transport_cancel_pkt()
229 		struct virtqueue *rx_vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_cancel_pkt()
232 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in virtio_transport_cancel_pkt()
235 			queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_transport_cancel_pkt()
245 static void virtio_vsock_rx_fill(struct virtio_vsock *vsock)  in virtio_vsock_rx_fill()  argument
253 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_vsock_rx_fill()
279 		vsock->rx_buf_nr++;  in virtio_vsock_rx_fill()
281 	if (vsock->rx_buf_nr > vsock->rx_buf_max_nr)  in virtio_vsock_rx_fill()
282 		vsock->rx_buf_max_nr = vsock->rx_buf_nr;  in virtio_vsock_rx_fill()
288 	struct virtio_vsock *vsock =  in virtio_transport_tx_work()  local
293 	vq = vsock->vqs[VSOCK_VQ_TX];  in virtio_transport_tx_work()
294 	mutex_lock(&vsock->tx_lock);  in virtio_transport_tx_work()
296 	if (!vsock->tx_run)  in virtio_transport_tx_work()
311 	mutex_unlock(&vsock->tx_lock);  in virtio_transport_tx_work()
314 		queue_work(virtio_vsock_workqueue, &vsock->send_pkt_work);  in virtio_transport_tx_work()
318 static bool virtio_transport_more_replies(struct virtio_vsock *vsock)  in virtio_transport_more_replies()  argument
320 	struct virtqueue *vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_more_replies()
324 	val = atomic_read(&vsock->queued_replies);  in virtio_transport_more_replies()
330 static int virtio_vsock_event_fill_one(struct virtio_vsock *vsock,  in virtio_vsock_event_fill_one()  argument
336 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_vsock_event_fill_one()
344 static void virtio_vsock_event_fill(struct virtio_vsock *vsock)  in virtio_vsock_event_fill()  argument
348 	for (i = 0; i < ARRAY_SIZE(vsock->event_list); i++) {  in virtio_vsock_event_fill()
349 		struct virtio_vsock_event *event = &vsock->event_list[i];  in virtio_vsock_event_fill()
351 		virtio_vsock_event_fill_one(vsock, event);  in virtio_vsock_event_fill()
354 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_vsock_event_fill()
366 static void virtio_vsock_update_guest_cid(struct virtio_vsock *vsock)  in virtio_vsock_update_guest_cid()  argument
368 	struct virtio_device *vdev = vsock->vdev;  in virtio_vsock_update_guest_cid()
373 	vsock->guest_cid = le64_to_cpu(guest_cid);  in virtio_vsock_update_guest_cid()
377 static void virtio_vsock_event_handle(struct virtio_vsock *vsock,  in virtio_vsock_event_handle()  argument
382 		virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_event_handle()
390 	struct virtio_vsock *vsock =  in virtio_transport_event_work()  local
394 	vq = vsock->vqs[VSOCK_VQ_EVENT];  in virtio_transport_event_work()
396 	mutex_lock(&vsock->event_lock);  in virtio_transport_event_work()
398 	if (!vsock->event_run)  in virtio_transport_event_work()
408 				virtio_vsock_event_handle(vsock, event);  in virtio_transport_event_work()
410 			virtio_vsock_event_fill_one(vsock, event);  in virtio_transport_event_work()
414 	virtqueue_kick(vsock->vqs[VSOCK_VQ_EVENT]);  in virtio_transport_event_work()
416 	mutex_unlock(&vsock->event_lock);  in virtio_transport_event_work()
421 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_event_done()  local
423 	if (!vsock)  in virtio_vsock_event_done()
425 	queue_work(virtio_vsock_workqueue, &vsock->event_work);  in virtio_vsock_event_done()
430 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_tx_done()  local
432 	if (!vsock)  in virtio_vsock_tx_done()
434 	queue_work(virtio_vsock_workqueue, &vsock->tx_work);  in virtio_vsock_tx_done()
439 	struct virtio_vsock *vsock = vq->vdev->priv;  in virtio_vsock_rx_done()  local
441 	if (!vsock)  in virtio_vsock_rx_done()
443 	queue_work(virtio_vsock_workqueue, &vsock->rx_work);  in virtio_vsock_rx_done()
490 	struct virtio_vsock *vsock =  in virtio_transport_rx_work()  local
494 	vq = vsock->vqs[VSOCK_VQ_RX];  in virtio_transport_rx_work()
496 	mutex_lock(&vsock->rx_lock);  in virtio_transport_rx_work()
498 	if (!vsock->rx_run)  in virtio_transport_rx_work()
507 			if (!virtio_transport_more_replies(vsock)) {  in virtio_transport_rx_work()
520 			vsock->rx_buf_nr--;  in virtio_transport_rx_work()
536 	if (vsock->rx_buf_nr < vsock->rx_buf_max_nr / 2)  in virtio_transport_rx_work()
537 		virtio_vsock_rx_fill(vsock);  in virtio_transport_rx_work()
538 	mutex_unlock(&vsock->rx_lock);  in virtio_transport_rx_work()
553 	struct virtio_vsock *vsock = NULL;  in virtio_vsock_probe()  local
567 	vsock = kzalloc(sizeof(*vsock), GFP_KERNEL);  in virtio_vsock_probe()
568 	if (!vsock) {  in virtio_vsock_probe()
573 	vsock->vdev = vdev;  in virtio_vsock_probe()
575 	ret = virtio_find_vqs(vsock->vdev, VSOCK_VQ_MAX,  in virtio_vsock_probe()
576 			      vsock->vqs, callbacks, names,  in virtio_vsock_probe()
581 	virtio_vsock_update_guest_cid(vsock);  in virtio_vsock_probe()
583 	vsock->rx_buf_nr = 0;  in virtio_vsock_probe()
584 	vsock->rx_buf_max_nr = 0;  in virtio_vsock_probe()
585 	atomic_set(&vsock->queued_replies, 0);  in virtio_vsock_probe()
587 	mutex_init(&vsock->tx_lock);  in virtio_vsock_probe()
588 	mutex_init(&vsock->rx_lock);  in virtio_vsock_probe()
589 	mutex_init(&vsock->event_lock);  in virtio_vsock_probe()
590 	spin_lock_init(&vsock->send_pkt_list_lock);  in virtio_vsock_probe()
591 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in virtio_vsock_probe()
592 	INIT_WORK(&vsock->rx_work, virtio_transport_rx_work);  in virtio_vsock_probe()
593 	INIT_WORK(&vsock->tx_work, virtio_transport_tx_work);  in virtio_vsock_probe()
594 	INIT_WORK(&vsock->event_work, virtio_transport_event_work);  in virtio_vsock_probe()
595 	INIT_WORK(&vsock->send_pkt_work, virtio_transport_send_pkt_work);  in virtio_vsock_probe()
597 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_probe()
598 	vsock->tx_run = true;  in virtio_vsock_probe()
599 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_probe()
601 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_probe()
602 	virtio_vsock_rx_fill(vsock);  in virtio_vsock_probe()
603 	vsock->rx_run = true;  in virtio_vsock_probe()
604 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_probe()
606 	mutex_lock(&vsock->event_lock);  in virtio_vsock_probe()
607 	virtio_vsock_event_fill(vsock);  in virtio_vsock_probe()
608 	vsock->event_run = true;  in virtio_vsock_probe()
609 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_probe()
611 	vdev->priv = vsock;  in virtio_vsock_probe()
612 	rcu_assign_pointer(the_virtio_vsock, vsock);  in virtio_vsock_probe()
618 	kfree(vsock);  in virtio_vsock_probe()
625 	struct virtio_vsock *vsock = vdev->priv;  in virtio_vsock_remove()  local
640 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
641 	vsock->rx_run = false;  in virtio_vsock_remove()
642 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
644 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
645 	vsock->tx_run = false;  in virtio_vsock_remove()
646 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
648 	mutex_lock(&vsock->event_lock);  in virtio_vsock_remove()
649 	vsock->event_run = false;  in virtio_vsock_remove()
650 	mutex_unlock(&vsock->event_lock);  in virtio_vsock_remove()
657 	mutex_lock(&vsock->rx_lock);  in virtio_vsock_remove()
658 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_RX])))  in virtio_vsock_remove()
660 	mutex_unlock(&vsock->rx_lock);  in virtio_vsock_remove()
662 	mutex_lock(&vsock->tx_lock);  in virtio_vsock_remove()
663 	while ((pkt = virtqueue_detach_unused_buf(vsock->vqs[VSOCK_VQ_TX])))  in virtio_vsock_remove()
665 	mutex_unlock(&vsock->tx_lock);  in virtio_vsock_remove()
667 	spin_lock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
668 	while (!list_empty(&vsock->send_pkt_list)) {  in virtio_vsock_remove()
669 		pkt = list_first_entry(&vsock->send_pkt_list,  in virtio_vsock_remove()
674 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in virtio_vsock_remove()
682 	flush_work(&vsock->rx_work);  in virtio_vsock_remove()
683 	flush_work(&vsock->tx_work);  in virtio_vsock_remove()
684 	flush_work(&vsock->event_work);  in virtio_vsock_remove()
685 	flush_work(&vsock->send_pkt_work);  in virtio_vsock_remove()
689 	kfree(vsock);  in virtio_vsock_remove()