73 	struct vhost_vsock *vsock;  in vhost_vsock_get()  local
75 	hash_for_each_possible_rcu(vhost_vsock_hash, vsock, hash, guest_cid) {  in vhost_vsock_get()
76 		u32 other_cid = vsock->guest_cid;  in vhost_vsock_get()
83 			return vsock;  in vhost_vsock_get()
91 vhost_transport_do_send_pkt(struct vhost_vsock *vsock,  in vhost_transport_do_send_pkt()  argument
94 	struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_transport_do_send_pkt()
108 	vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
119 		spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
120 		if (list_empty(&vsock->send_pkt_list)) {  in vhost_transport_do_send_pkt()
121 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
122 			vhost_enable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
126 		pkt = list_first_entry(&vsock->send_pkt_list,  in vhost_transport_do_send_pkt()
129 		spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
134 			spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
135 			list_add(&pkt->list, &vsock->send_pkt_list);  in vhost_transport_do_send_pkt()
136 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
141 			spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
142 			list_add(&pkt->list, &vsock->send_pkt_list);  in vhost_transport_do_send_pkt()
143 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
148 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_transport_do_send_pkt()
149 				vhost_disable_notify(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
240 			spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
241 			list_add(&pkt->list, &vsock->send_pkt_list);  in vhost_transport_do_send_pkt()
242 			spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_do_send_pkt()
247 				val = atomic_dec_return(&vsock->queued_replies);  in vhost_transport_do_send_pkt()
260 		vhost_signal(&vsock->dev, vq);  in vhost_transport_do_send_pkt()
272 	struct vhost_vsock *vsock;  in vhost_transport_send_pkt_work()  local
274 	vsock = container_of(work, struct vhost_vsock, send_pkt_work);  in vhost_transport_send_pkt_work()
275 	vq = &vsock->vqs[VSOCK_VQ_RX];  in vhost_transport_send_pkt_work()
277 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_transport_send_pkt_work()
283 	struct vhost_vsock *vsock;  in vhost_transport_send_pkt()  local
289 	vsock = vhost_vsock_get(le64_to_cpu(pkt->hdr.dst_cid));  in vhost_transport_send_pkt()
290 	if (!vsock) {  in vhost_transport_send_pkt()
297 		atomic_inc(&vsock->queued_replies);  in vhost_transport_send_pkt()
299 	spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_send_pkt()
300 	list_add_tail(&pkt->list, &vsock->send_pkt_list);  in vhost_transport_send_pkt()
301 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_send_pkt()
303 	vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);  in vhost_transport_send_pkt()
312 	struct vhost_vsock *vsock;  in vhost_transport_cancel_pkt()  local
321 	vsock = vhost_vsock_get(vsk->remote_addr.svm_cid);  in vhost_transport_cancel_pkt()
322 	if (!vsock)  in vhost_transport_cancel_pkt()
325 	spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_cancel_pkt()
326 	list_for_each_entry_safe(pkt, n, &vsock->send_pkt_list, list) {  in vhost_transport_cancel_pkt()
331 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_transport_cancel_pkt()
341 		struct vhost_virtqueue *tx_vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_transport_cancel_pkt()
344 		new_cnt = atomic_sub_return(cnt, &vsock->queued_replies);  in vhost_transport_cancel_pkt()
416 static bool vhost_vsock_more_replies(struct vhost_vsock *vsock)  in vhost_vsock_more_replies()  argument
418 	struct vhost_virtqueue *vq = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_more_replies()
422 	val = atomic_read(&vsock->queued_replies);  in vhost_vsock_more_replies()
479 	struct vhost_vsock *vsock;  in vhost_transport_seqpacket_allow()  local
483 	vsock = vhost_vsock_get(remote_cid);  in vhost_transport_seqpacket_allow()
485 	if (vsock)  in vhost_transport_seqpacket_allow()
486 		seqpacket_allow = vsock->seqpacket_allow;  in vhost_transport_seqpacket_allow()
497 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_tx_kick()  local
512 	vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
514 		if (!vhost_vsock_more_replies(vsock)) {  in vhost_vsock_handle_tx_kick()
528 			if (unlikely(vhost_enable_notify(&vsock->dev, vq))) {  in vhost_vsock_handle_tx_kick()
529 				vhost_disable_notify(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
547 		if (le64_to_cpu(pkt->hdr.src_cid) == vsock->guest_cid &&  in vhost_vsock_handle_tx_kick()
560 		vhost_signal(&vsock->dev, vq);  in vhost_vsock_handle_tx_kick()
570 	struct vhost_vsock *vsock = container_of(vq->dev, struct vhost_vsock,  in vhost_vsock_handle_rx_kick()  local
573 	vhost_transport_do_send_pkt(vsock, vq);  in vhost_vsock_handle_rx_kick()
576 static int vhost_vsock_start(struct vhost_vsock *vsock)  in vhost_vsock_start()  argument
582 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_start()
584 	ret = vhost_dev_check_owner(&vsock->dev);  in vhost_vsock_start()
588 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_start()
589 		vq = &vsock->vqs[i];  in vhost_vsock_start()
599 			vhost_vq_set_backend(vq, vsock);  in vhost_vsock_start()
611 	vhost_work_queue(&vsock->dev, &vsock->send_pkt_work);  in vhost_vsock_start()
613 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_start()
620 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_start()
621 		vq = &vsock->vqs[i];  in vhost_vsock_start()
628 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_start()
632 static int vhost_vsock_stop(struct vhost_vsock *vsock, bool check_owner)  in vhost_vsock_stop()  argument
637 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_stop()
640 		ret = vhost_dev_check_owner(&vsock->dev);  in vhost_vsock_stop()
645 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_stop()
646 		struct vhost_virtqueue *vq = &vsock->vqs[i];  in vhost_vsock_stop()
654 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_stop()
658 static void vhost_vsock_free(struct vhost_vsock *vsock)  in vhost_vsock_free()  argument
660 	kvfree(vsock);  in vhost_vsock_free()
666 	struct vhost_vsock *vsock;  in vhost_vsock_dev_open()  local
672 	vsock = kvmalloc(sizeof(*vsock), GFP_KERNEL | __GFP_RETRY_MAYFAIL);  in vhost_vsock_dev_open()
673 	if (!vsock)  in vhost_vsock_dev_open()
676 	vqs = kmalloc_array(ARRAY_SIZE(vsock->vqs), sizeof(*vqs), GFP_KERNEL);  in vhost_vsock_dev_open()
682 	vsock->guest_cid = 0; /* no CID assigned yet */  in vhost_vsock_dev_open()
684 	atomic_set(&vsock->queued_replies, 0);  in vhost_vsock_dev_open()
686 	vqs[VSOCK_VQ_TX] = &vsock->vqs[VSOCK_VQ_TX];  in vhost_vsock_dev_open()
687 	vqs[VSOCK_VQ_RX] = &vsock->vqs[VSOCK_VQ_RX];  in vhost_vsock_dev_open()
688 	vsock->vqs[VSOCK_VQ_TX].handle_kick = vhost_vsock_handle_tx_kick;  in vhost_vsock_dev_open()
689 	vsock->vqs[VSOCK_VQ_RX].handle_kick = vhost_vsock_handle_rx_kick;  in vhost_vsock_dev_open()
691 	vhost_dev_init(&vsock->dev, vqs, ARRAY_SIZE(vsock->vqs),  in vhost_vsock_dev_open()
695 	file->private_data = vsock;  in vhost_vsock_dev_open()
696 	spin_lock_init(&vsock->send_pkt_list_lock);  in vhost_vsock_dev_open()
697 	INIT_LIST_HEAD(&vsock->send_pkt_list);  in vhost_vsock_dev_open()
698 	vhost_work_init(&vsock->send_pkt_work, vhost_transport_send_pkt_work);  in vhost_vsock_dev_open()
702 	vhost_vsock_free(vsock);  in vhost_vsock_dev_open()
706 static void vhost_vsock_flush(struct vhost_vsock *vsock)  in vhost_vsock_flush()  argument
708 	vhost_dev_flush(&vsock->dev);  in vhost_vsock_flush()
739 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_dev_release()  local
742 	if (vsock->guest_cid)  in vhost_vsock_dev_release()
743 		hash_del_rcu(&vsock->hash);  in vhost_vsock_dev_release()
759 	vhost_vsock_stop(vsock, false);  in vhost_vsock_dev_release()
760 	vhost_vsock_flush(vsock);  in vhost_vsock_dev_release()
761 	vhost_dev_stop(&vsock->dev);  in vhost_vsock_dev_release()
763 	spin_lock_bh(&vsock->send_pkt_list_lock);  in vhost_vsock_dev_release()
764 	while (!list_empty(&vsock->send_pkt_list)) {  in vhost_vsock_dev_release()
767 		pkt = list_first_entry(&vsock->send_pkt_list,  in vhost_vsock_dev_release()
772 	spin_unlock_bh(&vsock->send_pkt_list_lock);  in vhost_vsock_dev_release()
774 	vhost_dev_cleanup(&vsock->dev);  in vhost_vsock_dev_release()
775 	kfree(vsock->dev.vqs);  in vhost_vsock_dev_release()
776 	vhost_vsock_free(vsock);  in vhost_vsock_dev_release()
780 static int vhost_vsock_set_cid(struct vhost_vsock *vsock, u64 guest_cid)  in vhost_vsock_set_cid()  argument
802 	if (other && other != vsock) {  in vhost_vsock_set_cid()
807 	if (vsock->guest_cid)  in vhost_vsock_set_cid()
808 		hash_del_rcu(&vsock->hash);  in vhost_vsock_set_cid()
810 	vsock->guest_cid = guest_cid;  in vhost_vsock_set_cid()
811 	hash_add_rcu(vhost_vsock_hash, &vsock->hash, vsock->guest_cid);  in vhost_vsock_set_cid()
817 static int vhost_vsock_set_features(struct vhost_vsock *vsock, u64 features)  in vhost_vsock_set_features()  argument
825 	mutex_lock(&vsock->dev.mutex);  in vhost_vsock_set_features()
827 	    !vhost_log_access_ok(&vsock->dev)) {  in vhost_vsock_set_features()
832 		if (vhost_init_device_iotlb(&vsock->dev, true))  in vhost_vsock_set_features()
837 		vsock->seqpacket_allow = true;  in vhost_vsock_set_features()
839 	for (i = 0; i < ARRAY_SIZE(vsock->vqs); i++) {  in vhost_vsock_set_features()
840 		vq = &vsock->vqs[i];  in vhost_vsock_set_features()
845 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_set_features()
849 	mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_set_features()
856 	struct vhost_vsock *vsock = f->private_data;  in vhost_vsock_dev_ioctl()  local
867 		return vhost_vsock_set_cid(vsock, guest_cid);  in vhost_vsock_dev_ioctl()
872 			return vhost_vsock_start(vsock);  in vhost_vsock_dev_ioctl()
874 			return vhost_vsock_stop(vsock, true);  in vhost_vsock_dev_ioctl()
883 		return vhost_vsock_set_features(vsock, features);  in vhost_vsock_dev_ioctl()
894 		vhost_set_backend_features(&vsock->dev, features);  in vhost_vsock_dev_ioctl()
897 		mutex_lock(&vsock->dev.mutex);  in vhost_vsock_dev_ioctl()
898 		r = vhost_dev_ioctl(&vsock->dev, ioctl, argp);  in vhost_vsock_dev_ioctl()
900 			r = vhost_vring_ioctl(&vsock->dev, ioctl, argp);  in vhost_vsock_dev_ioctl()
902 			vhost_vsock_flush(vsock);  in vhost_vsock_dev_ioctl()
903 		mutex_unlock(&vsock->dev.mutex);  in vhost_vsock_dev_ioctl()
911 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_read_iter()  local
912 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_read_iter()
922 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_write_iter()  local
923 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_write_iter()
930 	struct vhost_vsock *vsock = file->private_data;  in vhost_vsock_chr_poll()  local
931 	struct vhost_dev *dev = &vsock->dev;  in vhost_vsock_chr_poll()