107 	struct virtproc_info *vrp;  member
152 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,
208 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,  in __rpmsg_create_ept()  argument
215 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
238 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
241 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
248 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
253 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
262 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()  local
264 	return __rpmsg_create_channel(vrp, chinfo);  in virtio_rpmsg_create_channel()
271 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()  local
273 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
283 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
297 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)  in __rpmsg_destroy_ept()  argument
300 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
301 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
302 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
316 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
322 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()  local
328 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
346 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()  local
352 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
388 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,  in __rpmsg_create_channel()  argument
393 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
411 	vch->vrp = vrp;  in __rpmsg_create_channel()
418 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
428 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
438 static void *get_a_tx_buf(struct virtproc_info *vrp)  in get_a_tx_buf()  argument
444 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
450 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
451 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
454 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
456 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
477 static void rpmsg_upref_sleepers(struct virtproc_info *vrp)  in rpmsg_upref_sleepers()  argument
480 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
483 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
485 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
487 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
504 static void rpmsg_downref_sleepers(struct virtproc_info *vrp)  in rpmsg_downref_sleepers()  argument
507 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
510 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
512 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
514 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
556 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()  local
577 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
583 	msg = get_a_tx_buf(vrp);  in rpmsg_send_offchannel_raw()
590 		rpmsg_upref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
598 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
599 					(msg = get_a_tx_buf(vrp)),  in rpmsg_send_offchannel_raw()
603 		rpmsg_downref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
628 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
631 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
643 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
645 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
699 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,  in rpmsg_recv_single()  argument
704 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
722 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
729 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
731 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
737 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
755 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
758 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
770 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()  local
783 		err = rpmsg_recv_single(vrp, dev, msg, len);  in rpmsg_recv_done()
796 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
808 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()  local
813 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
823 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()  local
833 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
839 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
841 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
864 	struct virtproc_info *vrp;  in rpmsg_probe()  local
872 	vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);  in rpmsg_probe()
873 	if (!vrp)  in rpmsg_probe()
876 	vrp->vdev = vdev;  in rpmsg_probe()
878 	idr_init(&vrp->endpoints);  in rpmsg_probe()
879 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
880 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
881 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
888 	vrp->rvq = vqs[0];  in rpmsg_probe()
889 	vrp->svq = vqs[1];  in rpmsg_probe()
892 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
893 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
896 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
897 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
899 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
901 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
903 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
907 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
915 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
918 	vrp->rbufs = bufs_va;  in rpmsg_probe()
921 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
924 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
926 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
928 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
930 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
936 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
938 	vdev->priv = vrp;  in rpmsg_probe()
955 		vch->vrp = vrp;  in rpmsg_probe()
960 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
962 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
974 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
985 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
997 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
999 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1001 	kfree(vrp);  in rpmsg_probe()
1014 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()  local
1015 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1024 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1026 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1029 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1031 	kfree(vrp);  in rpmsg_remove()