130 	struct virtproc_info *vrp;  member
232 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,  in __rpmsg_create_ept()  argument
239 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
262 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
265 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
272 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
277 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
289 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
303 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)  in __rpmsg_destroy_ept()  argument
306 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
307 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
308 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
322 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
328 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()  local
334 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
352 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()  local
358 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
392 static struct rpmsg_device *rpmsg_create_channel(struct virtproc_info *vrp,  in rpmsg_create_channel()  argument
397 	struct device *tmp, *dev = &vrp->vdev->dev;  in rpmsg_create_channel()
415 	vch->vrp = vrp;  in rpmsg_create_channel()
431 	rpdev->dev.parent = &vrp->vdev->dev;  in rpmsg_create_channel()
441 static void *get_a_tx_buf(struct virtproc_info *vrp)  in get_a_tx_buf()  argument
447 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
453 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
454 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
457 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
459 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
480 static void rpmsg_upref_sleepers(struct virtproc_info *vrp)  in rpmsg_upref_sleepers()  argument
483 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
486 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
488 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
490 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
507 static void rpmsg_downref_sleepers(struct virtproc_info *vrp)  in rpmsg_downref_sleepers()  argument
510 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
513 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
515 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
517 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
559 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()  local
580 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
586 	msg = get_a_tx_buf(vrp);  in rpmsg_send_offchannel_raw()
593 		rpmsg_upref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
601 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
602 					(msg = get_a_tx_buf(vrp)),  in rpmsg_send_offchannel_raw()
606 		rpmsg_downref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
631 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
634 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
646 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
648 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
702 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,  in rpmsg_recv_single()  argument
720 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
727 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
729 	ept = idr_find(&vrp->endpoints, msg->dst);  in rpmsg_recv_single()
735 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
753 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
756 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
768 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()  local
781 		err = rpmsg_recv_single(vrp, dev, msg, len);  in rpmsg_recv_done()
794 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
806 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()  local
811 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
821 	struct virtproc_info *vrp = priv;  in rpmsg_ns_cb()  local
822 	struct device *dev = &vrp->vdev->dev;  in rpmsg_ns_cb()
858 		ret = rpmsg_unregister_device(&vrp->vdev->dev, &chinfo);  in rpmsg_ns_cb()
862 		newch = rpmsg_create_channel(vrp, &chinfo);  in rpmsg_ns_cb()
875 	struct virtproc_info *vrp;  in rpmsg_probe()  local
881 	vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);  in rpmsg_probe()
882 	if (!vrp)  in rpmsg_probe()
885 	vrp->vdev = vdev;  in rpmsg_probe()
887 	idr_init(&vrp->endpoints);  in rpmsg_probe()
888 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
889 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
890 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
897 	vrp->rvq = vqs[0];  in rpmsg_probe()
898 	vrp->svq = vqs[1];  in rpmsg_probe()
901 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
902 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
905 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
906 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
908 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
910 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
912 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
916 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
924 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
927 	vrp->rbufs = bufs_va;  in rpmsg_probe()
930 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
933 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
935 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
937 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
939 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
945 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
947 	vdev->priv = vrp;  in rpmsg_probe()
952 		vrp->ns_ept = __rpmsg_create_ept(vrp, NULL, rpmsg_ns_cb,  in rpmsg_probe()
953 						vrp, RPMSG_NS_ADDR);  in rpmsg_probe()
954 		if (!vrp->ns_ept) {  in rpmsg_probe()
965 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
976 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
984 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
986 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
988 	kfree(vrp);  in rpmsg_probe()
1001 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()  local
1002 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1011 	if (vrp->ns_ept)  in rpmsg_remove()
1012 		__rpmsg_destroy_ept(vrp, vrp->ns_ept);  in rpmsg_remove()
1014 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1016 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1019 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1021 	kfree(vrp);  in rpmsg_remove()