106 	struct virtproc_info *vrp;  member
152 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,
209 static struct rpmsg_endpoint *__rpmsg_create_ept(struct virtproc_info *vrp,  in __rpmsg_create_ept()  argument
216 	struct device *dev = rpdev ? &rpdev->dev : &vrp->vdev->dev;  in __rpmsg_create_ept()
239 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
242 	id = idr_alloc(&vrp->endpoints, ept, id_min, id_max, GFP_KERNEL);  in __rpmsg_create_ept()
249 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
254 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_create_ept()
263 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_create_channel()  local
265 	return __rpmsg_create_channel(vrp, chinfo);  in virtio_rpmsg_create_channel()
272 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_release_channel()  local
274 	return rpmsg_unregister_device(&vrp->vdev->dev, chinfo);  in virtio_rpmsg_release_channel()
284 	return __rpmsg_create_ept(vch->vrp, rpdev, cb, priv, chinfo.src);  in virtio_rpmsg_create_ept()
298 __rpmsg_destroy_ept(struct virtproc_info *vrp, struct rpmsg_endpoint *ept)  in __rpmsg_destroy_ept()  argument
301 	mutex_lock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
302 	idr_remove(&vrp->endpoints, ept->addr);  in __rpmsg_destroy_ept()
303 	mutex_unlock(&vrp->endpoints_lock);  in __rpmsg_destroy_ept()
317 	__rpmsg_destroy_ept(vch->vrp, ept);  in virtio_rpmsg_destroy_ept()
323 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_create()  local
329 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_create()
347 	struct virtproc_info *vrp = vch->vrp;  in virtio_rpmsg_announce_destroy()  local
353 	    virtio_has_feature(vrp->vdev, VIRTIO_RPMSG_F_NS)) {  in virtio_rpmsg_announce_destroy()
389 static struct rpmsg_device *__rpmsg_create_channel(struct virtproc_info *vrp,  in __rpmsg_create_channel()  argument
394 	struct device *tmp, *dev = &vrp->vdev->dev;  in __rpmsg_create_channel()
412 	vch->vrp = vrp;  in __rpmsg_create_channel()
419 	rpdev->little_endian = virtio_is_little_endian(vrp->vdev);  in __rpmsg_create_channel()
429 	rpdev->dev.parent = &vrp->vdev->dev;  in __rpmsg_create_channel()
439 static void *get_a_tx_buf(struct virtproc_info *vrp)  in get_a_tx_buf()  argument
445 	mutex_lock(&vrp->tx_lock);  in get_a_tx_buf()
451 	if (vrp->last_sbuf < vrp->num_bufs / 2)  in get_a_tx_buf()
452 		ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++;  in get_a_tx_buf()
455 		ret = virtqueue_get_buf(vrp->svq, &len);  in get_a_tx_buf()
457 	mutex_unlock(&vrp->tx_lock);  in get_a_tx_buf()
478 static void rpmsg_upref_sleepers(struct virtproc_info *vrp)  in rpmsg_upref_sleepers()  argument
481 	mutex_lock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
484 	if (atomic_inc_return(&vrp->sleepers) == 1)  in rpmsg_upref_sleepers()
486 		virtqueue_enable_cb(vrp->svq);  in rpmsg_upref_sleepers()
488 	mutex_unlock(&vrp->tx_lock);  in rpmsg_upref_sleepers()
505 static void rpmsg_downref_sleepers(struct virtproc_info *vrp)  in rpmsg_downref_sleepers()  argument
508 	mutex_lock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
511 	if (atomic_dec_and_test(&vrp->sleepers))  in rpmsg_downref_sleepers()
513 		virtqueue_disable_cb(vrp->svq);  in rpmsg_downref_sleepers()
515 	mutex_unlock(&vrp->tx_lock);  in rpmsg_downref_sleepers()
557 	struct virtproc_info *vrp = vch->vrp;  in rpmsg_send_offchannel_raw()  local
578 	if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) {  in rpmsg_send_offchannel_raw()
584 	msg = get_a_tx_buf(vrp);  in rpmsg_send_offchannel_raw()
591 		rpmsg_upref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
599 		err = wait_event_interruptible_timeout(vrp->sendq,  in rpmsg_send_offchannel_raw()
600 					(msg = get_a_tx_buf(vrp)),  in rpmsg_send_offchannel_raw()
604 		rpmsg_downref_sleepers(vrp);  in rpmsg_send_offchannel_raw()
629 	mutex_lock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
632 	err = virtqueue_add_outbuf(vrp->svq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_send_offchannel_raw()
644 	virtqueue_kick(vrp->svq);  in rpmsg_send_offchannel_raw()
646 	mutex_unlock(&vrp->tx_lock);  in rpmsg_send_offchannel_raw()
705 	return vch->vrp->buf_size - sizeof(struct rpmsg_hdr);  in virtio_rpmsg_get_mtu()
708 static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev,  in rpmsg_recv_single()  argument
713 	bool little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_recv_single()
731 	if (len > vrp->buf_size ||  in rpmsg_recv_single()
738 	mutex_lock(&vrp->endpoints_lock);  in rpmsg_recv_single()
740 	ept = idr_find(&vrp->endpoints, __rpmsg32_to_cpu(little_endian, msg->dst));  in rpmsg_recv_single()
746 	mutex_unlock(&vrp->endpoints_lock);  in rpmsg_recv_single()
764 	rpmsg_sg_init(&sg, msg, vrp->buf_size);  in rpmsg_recv_single()
767 	err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL);  in rpmsg_recv_single()
779 	struct virtproc_info *vrp = rvq->vdev->priv;  in rpmsg_recv_done()  local
792 		err = rpmsg_recv_single(vrp, dev, msg, len);  in rpmsg_recv_done()
805 		virtqueue_kick(vrp->rvq);  in rpmsg_recv_done()
817 	struct virtproc_info *vrp = svq->vdev->priv;  in rpmsg_xmit_done()  local
822 	wake_up_interruptible(&vrp->sendq);  in rpmsg_xmit_done()
832 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_virtio_add_ctrl_dev()  local
842 	vch->vrp = vrp;  in rpmsg_virtio_add_ctrl_dev()
848 	rpdev_ctrl->dev.parent = &vrp->vdev->dev;  in rpmsg_virtio_add_ctrl_dev()
850 	rpdev_ctrl->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_virtio_add_ctrl_dev()
873 	struct virtproc_info *vrp;  in rpmsg_probe()  local
881 	vrp = kzalloc(sizeof(*vrp), GFP_KERNEL);  in rpmsg_probe()
882 	if (!vrp)  in rpmsg_probe()
885 	vrp->vdev = vdev;  in rpmsg_probe()
887 	idr_init(&vrp->endpoints);  in rpmsg_probe()
888 	mutex_init(&vrp->endpoints_lock);  in rpmsg_probe()
889 	mutex_init(&vrp->tx_lock);  in rpmsg_probe()
890 	init_waitqueue_head(&vrp->sendq);  in rpmsg_probe()
897 	vrp->rvq = vqs[0];  in rpmsg_probe()
898 	vrp->svq = vqs[1];  in rpmsg_probe()
901 	WARN_ON(virtqueue_get_vring_size(vrp->rvq) !=  in rpmsg_probe()
902 		virtqueue_get_vring_size(vrp->svq));  in rpmsg_probe()
905 	if (virtqueue_get_vring_size(vrp->rvq) < MAX_RPMSG_NUM_BUFS / 2)  in rpmsg_probe()
906 		vrp->num_bufs = virtqueue_get_vring_size(vrp->rvq) * 2;  in rpmsg_probe()
908 		vrp->num_bufs = MAX_RPMSG_NUM_BUFS;  in rpmsg_probe()
910 	vrp->buf_size = MAX_RPMSG_BUF_SIZE;  in rpmsg_probe()
912 	total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_probe()
916 				     total_buf_space, &vrp->bufs_dma,  in rpmsg_probe()
924 		bufs_va, &vrp->bufs_dma);  in rpmsg_probe()
927 	vrp->rbufs = bufs_va;  in rpmsg_probe()
930 	vrp->sbufs = bufs_va + total_buf_space / 2;  in rpmsg_probe()
933 	for (i = 0; i < vrp->num_bufs / 2; i++) {  in rpmsg_probe()
935 		void *cpu_addr = vrp->rbufs + i * vrp->buf_size;  in rpmsg_probe()
937 		rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size);  in rpmsg_probe()
939 		err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr,  in rpmsg_probe()
945 	virtqueue_disable_cb(vrp->svq);  in rpmsg_probe()
947 	vdev->priv = vrp;  in rpmsg_probe()
964 		vch->vrp = vrp;  in rpmsg_probe()
969 		rpdev_ns->little_endian = virtio_is_little_endian(vrp->vdev);  in rpmsg_probe()
971 		rpdev_ns->dev.parent = &vrp->vdev->dev;  in rpmsg_probe()
984 	notify = virtqueue_kick_prepare(vrp->rvq);  in rpmsg_probe()
995 		virtqueue_notify(vrp->rvq);  in rpmsg_probe()
1005 			  bufs_va, vrp->bufs_dma);  in rpmsg_probe()
1007 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_probe()
1009 	kfree(vrp);  in rpmsg_probe()
1022 	struct virtproc_info *vrp = vdev->priv;  in rpmsg_remove()  local
1023 	size_t total_buf_space = vrp->num_bufs * vrp->buf_size;  in rpmsg_remove()
1032 	idr_destroy(&vrp->endpoints);  in rpmsg_remove()
1034 	vdev->config->del_vqs(vrp->vdev);  in rpmsg_remove()
1037 			  vrp->rbufs, vrp->bufs_dma);  in rpmsg_remove()
1039 	kfree(vrp);  in rpmsg_remove()