20 static inline struct device *vop_dev(struct vop_vdev *vdev)  in vop_dev()  argument
22 	return vdev->vpdev->dev.parent;  in vop_dev()
26 static inline int vop_vdev_inited(struct vop_vdev *vdev)  in vop_vdev_inited()  argument
28 	if (!vdev)  in vop_vdev_inited()
31 	if (!vdev->dd || !vdev->dd->type) {  in vop_vdev_inited()
32 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_vdev_inited()
37 	if (vdev->dd->type == -1) {  in vop_vdev_inited()
38 		dev_dbg(vop_dev(vdev), "%s %d err %d\n",  in vop_vdev_inited()
48 	struct vop_vdev *vdev = vvrh->vdev;  in _vop_notify()  local
49 	struct vop_device *vpdev = vdev->vpdev;  in _vop_notify()
50 	s8 db = vdev->dc->h2c_vdev_db;  in _vop_notify()
56 static void vop_virtio_init_post(struct vop_vdev *vdev)  in vop_virtio_init_post()  argument
58 	struct mic_vqconfig *vqconfig = mic_vq_config(vdev->dd);  in vop_virtio_init_post()
59 	struct vop_device *vpdev = vdev->vpdev;  in vop_virtio_init_post()
62 	for (i = 0; i < vdev->dd->num_vq; i++) {  in vop_virtio_init_post()
67 			dev_warn(vop_dev(vdev), "used_address zero??\n");  in vop_virtio_init_post()
70 		vdev->vvr[i].vrh.vring.used =  in vop_virtio_init_post()
77 	vdev->dc->used_address_updated = 0;  in vop_virtio_init_post()
79 	dev_info(vop_dev(vdev), "%s: device type %d LINKUP\n",  in vop_virtio_init_post()
80 		 __func__, vdev->virtio_id);  in vop_virtio_init_post()
83 static inline void vop_virtio_device_reset(struct vop_vdev *vdev)  in vop_virtio_device_reset()  argument
87 	dev_dbg(vop_dev(vdev), "%s: status %d device type %d RESET\n",  in vop_virtio_device_reset()
88 		__func__, vdev->dd->status, vdev->virtio_id);  in vop_virtio_device_reset()
90 	for (i = 0; i < vdev->dd->num_vq; i++)  in vop_virtio_device_reset()
95 		mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1);  in vop_virtio_device_reset()
98 	vdev->dd->status = 0;  in vop_virtio_device_reset()
99 	vdev->dc->vdev_reset = 0;  in vop_virtio_device_reset()
100 	vdev->dc->host_ack = 1;  in vop_virtio_device_reset()
102 	for (i = 0; i < vdev->dd->num_vq; i++) {  in vop_virtio_device_reset()
103 		struct vringh *vrh = &vdev->vvr[i].vrh;  in vop_virtio_device_reset()
105 		vdev->vvr[i].vring.info->avail_idx = 0;  in vop_virtio_device_reset()
111 	for (i = 0; i < vdev->dd->num_vq; i++)  in vop_virtio_device_reset()
112 		mutex_unlock(&vdev->vvr[i].vr_mutex);  in vop_virtio_device_reset()
118 	struct vop_vdev *vdev;  in vop_virtio_reset_devices()  local
121 		vdev = list_entry(pos, struct vop_vdev, list);  in vop_virtio_reset_devices()
122 		vop_virtio_device_reset(vdev);  in vop_virtio_reset_devices()
123 		vdev->poll_wake = 1;  in vop_virtio_reset_devices()
124 		wake_up(&vdev->waitq);  in vop_virtio_reset_devices()
130 	struct vop_vdev *vdev = container_of(work, struct vop_vdev,  in vop_bh_handler()  local
133 	if (vdev->dc->used_address_updated)  in vop_bh_handler()
134 		vop_virtio_init_post(vdev);  in vop_bh_handler()
136 	if (vdev->dc->vdev_reset)  in vop_bh_handler()
137 		vop_virtio_device_reset(vdev);  in vop_bh_handler()
139 	vdev->poll_wake = 1;  in vop_bh_handler()
140 	wake_up(&vdev->waitq);  in vop_bh_handler()
145 	struct vop_vdev *vdev = data;  in _vop_virtio_intr_handler()  local
146 	struct vop_device *vpdev = vdev->vpdev;  in _vop_virtio_intr_handler()
148 	vpdev->hw_ops->ack_interrupt(vpdev, vdev->virtio_db);  in _vop_virtio_intr_handler()
149 	schedule_work(&vdev->virtio_bh_work);  in _vop_virtio_intr_handler()
153 static int vop_virtio_config_change(struct vop_vdev *vdev, void *argp)  in vop_virtio_config_change()  argument
157 	struct vop_device *vpdev = vdev->vpdev;  in vop_virtio_config_change()
163 	for (i = 0; i < vdev->dd->num_vq; i++)  in vop_virtio_config_change()
164 		mutex_lock_nested(&vdev->vvr[i].vr_mutex, i + 1);  in vop_virtio_config_change()
166 	if (db == -1 || vdev->dd->type == -1) {  in vop_virtio_config_change()
171 	memcpy(mic_vq_configspace(vdev->dd), argp, vdev->dd->config_len);  in vop_virtio_config_change()
172 	vdev->dc->config_change = MIC_VIRTIO_PARAM_CONFIG_CHANGED;  in vop_virtio_config_change()
176 		ret = wait_event_timeout(wake, vdev->dc->guest_ack,  in vop_virtio_config_change()
182 	dev_dbg(vop_dev(vdev),  in vop_virtio_config_change()
184 	vdev->dc->config_change = 0;  in vop_virtio_config_change()
185 	vdev->dc->guest_ack = 0;  in vop_virtio_config_change()
187 	for (i = 0; i < vdev->dd->num_vq; i++)  in vop_virtio_config_change()
188 		mutex_unlock(&vdev->vvr[i].vr_mutex);  in vop_virtio_config_change()
193 static int vop_copy_dp_entry(struct vop_vdev *vdev,  in vop_copy_dp_entry()  argument
197 	struct vop_device *vpdev = vdev->vpdev;  in vop_copy_dp_entry()
207 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_copy_dp_entry()
225 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_copy_dp_entry()
242 static void vop_init_device_ctrl(struct vop_vdev *vdev,  in vop_init_device_ctrl()  argument
256 	vdev->dc = dc;  in vop_init_device_ctrl()
259 static int vop_virtio_add_device(struct vop_vdev *vdev,  in vop_virtio_add_device()  argument
262 	struct vop_info *vi = vdev->vi;  in vop_virtio_add_device()
275 	init_waitqueue_head(&vdev->waitq);  in vop_virtio_add_device()
276 	INIT_LIST_HEAD(&vdev->list);  in vop_virtio_add_device()
277 	vdev->vpdev = vpdev;  in vop_virtio_add_device()
279 	ret = vop_copy_dp_entry(vdev, argp, &type, &dd);  in vop_virtio_add_device()
281 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_add_device()
286 	vop_init_device_ctrl(vdev, dd);  in vop_virtio_add_device()
288 	vdev->dd = dd;  in vop_virtio_add_device()
289 	vdev->virtio_id = type;  in vop_virtio_add_device()
291 	INIT_WORK(&vdev->virtio_bh_work, vop_bh_handler);  in vop_virtio_add_device()
294 		struct vop_vringh *vvr = &vdev->vvr[i];  in vop_virtio_add_device()
295 		struct mic_vring *vr = &vdev->vvr[i].vring;  in vop_virtio_add_device()
306 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_add_device()
312 		vr->info->magic = cpu_to_le32(MIC_MAGIC + vdev->virtio_id + i);  in vop_virtio_add_device()
318 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_add_device()
326 				       *(u32 *)mic_vq_features(vdev->dd),  in vop_virtio_add_device()
330 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_add_device()
337 		vvr->vdev = vdev;  in vop_virtio_add_device()
350 		 vdev->virtio_id);  in vop_virtio_add_device()
351 	vdev->virtio_db = vpdev->hw_ops->next_db(vpdev);  in vop_virtio_add_device()
352 	vdev->virtio_cookie = vpdev->hw_ops->request_irq(vpdev,  in vop_virtio_add_device()
353 			_vop_virtio_intr_handler, irqname, vdev,  in vop_virtio_add_device()
354 			vdev->virtio_db);  in vop_virtio_add_device()
355 	if (IS_ERR(vdev->virtio_cookie)) {  in vop_virtio_add_device()
356 		ret = PTR_ERR(vdev->virtio_cookie);  in vop_virtio_add_device()
361 	vdev->dc->c2h_vdev_db = vdev->virtio_db;  in vop_virtio_add_device()
383 		struct vop_vringh *vvr = &vdev->vvr[j];  in vop_virtio_add_device()
418 static void vop_virtio_del_device(struct vop_vdev *vdev)  in vop_virtio_del_device()  argument
420 	struct vop_info *vi = vdev->vi;  in vop_virtio_del_device()
421 	struct vop_device *vpdev = vdev->vpdev;  in vop_virtio_del_device()
428 	vop_dev_remove(vi, vdev->dc, vpdev);  in vop_virtio_del_device()
430 	vpdev->hw_ops->free_irq(vpdev, vdev->virtio_cookie, vdev);  in vop_virtio_del_device()
431 	flush_work(&vdev->virtio_bh_work);  in vop_virtio_del_device()
432 	vqconfig = mic_vq_config(vdev->dd);  in vop_virtio_del_device()
433 	for (i = 0; i < vdev->dd->num_vq; i++) {  in vop_virtio_del_device()
434 		struct vop_vringh *vvr = &vdev->vvr[i];  in vop_virtio_del_device()
455 	vdev->dd->type = -1;  in vop_virtio_del_device()
469 static int vop_sync_dma(struct vop_vdev *vdev, dma_addr_t dst, dma_addr_t src,  in vop_sync_dma()  argument
475 	struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev);  in vop_sync_dma()
514 static int vop_virtio_copy_to_user(struct vop_vdev *vdev, void __user *ubuf,  in vop_virtio_copy_to_user()  argument
518 	struct vop_device *vpdev = vdev->vpdev;  in vop_virtio_copy_to_user()
520 	struct vop_vringh *vvr = &vdev->vvr[vr_idx];  in vop_virtio_copy_to_user()
530 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_to_user()
534 		vdev->in_bytes += len;  in vop_virtio_copy_to_user()
555 		err = vop_sync_dma(vdev, vvr->buf_da, daddr,  in vop_virtio_copy_to_user()
558 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_to_user()
565 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_to_user()
572 		vdev->in_bytes_dma += partlen;  in vop_virtio_copy_to_user()
573 		vdev->in_bytes += partlen;  in vop_virtio_copy_to_user()
580 	dev_dbg(vop_dev(vdev),  in vop_virtio_copy_to_user()
592 static int vop_virtio_copy_from_user(struct vop_vdev *vdev, void __user *ubuf,  in vop_virtio_copy_from_user()  argument
596 	struct vop_device *vpdev = vdev->vpdev;  in vop_virtio_copy_from_user()
598 	struct vop_vringh *vvr = &vdev->vvr[vr_idx];  in vop_virtio_copy_from_user()
599 	struct vop_info *vi = dev_get_drvdata(&vdev->vpdev->dev);  in vop_virtio_copy_from_user()
611 			vdev->tx_dst_unaligned += len;  in vop_virtio_copy_from_user()
614 			vdev->tx_len_unaligned += len;  in vop_virtio_copy_from_user()
635 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_from_user()
639 		err = vop_sync_dma(vdev, daddr, vvr->buf_da,  in vop_virtio_copy_from_user()
642 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_from_user()
649 		vdev->out_bytes_dma += partlen;  in vop_virtio_copy_from_user()
650 		vdev->out_bytes += partlen;  in vop_virtio_copy_from_user()
660 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_from_user()
664 	vdev->out_bytes += len;  in vop_virtio_copy_from_user()
668 	dev_dbg(vop_dev(vdev),  in vop_virtio_copy_from_user()
694 static int vop_vringh_copy(struct vop_vdev *vdev, struct vringh_kiov *iov,  in vop_vringh_copy()  argument
707 			ret = vop_virtio_copy_to_user(vdev, ubuf, partlen,  in vop_vringh_copy()
712 			ret = vop_virtio_copy_from_user(vdev, ubuf, partlen,  in vop_vringh_copy()
717 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_vringh_copy()
744 static int _vop_virtio_copy(struct vop_vdev *vdev, struct mic_copy_desc *copy)  in _vop_virtio_copy()  argument
751 	struct vop_vringh *vvr = &vdev->vvr[copy->vr_idx];  in _vop_virtio_copy()
774 				dev_err(vop_dev(vdev), "%s %d err %d\n",  in _vop_virtio_copy()
782 		ret = vop_vringh_copy(vdev, riov, ubuf, len,  in _vop_virtio_copy()
785 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in _vop_virtio_copy()
793 		ret = vop_vringh_copy(vdev, wiov, ubuf, len,  in _vop_virtio_copy()
796 			dev_err(vop_dev(vdev), "%s %d err %d\n",  in _vop_virtio_copy()
834 static inline int vop_verify_copy_args(struct vop_vdev *vdev,  in vop_verify_copy_args()  argument
837 	if (!vdev || copy->vr_idx >= vdev->dd->num_vq)  in vop_verify_copy_args()
843 static int vop_virtio_copy_desc(struct vop_vdev *vdev,  in vop_virtio_copy_desc()  argument
849 	err = vop_verify_copy_args(vdev, copy);  in vop_virtio_copy_desc()
853 	vvr = &vdev->vvr[copy->vr_idx];  in vop_virtio_copy_desc()
855 	if (!vop_vdevup(vdev)) {  in vop_virtio_copy_desc()
857 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_desc()
861 	err = _vop_virtio_copy(vdev, copy);  in vop_virtio_copy_desc()
863 		dev_err(vop_dev(vdev), "%s %d err %d\n",  in vop_virtio_copy_desc()
873 	struct vop_vdev *vdev;  in vop_open()  local
877 	vdev = kzalloc(sizeof(*vdev), GFP_KERNEL);  in vop_open()
878 	if (!vdev)  in vop_open()
880 	vdev->vi = vi;  in vop_open()
881 	mutex_init(&vdev->vdev_mutex);  in vop_open()
882 	f->private_data = vdev;  in vop_open()
883 	init_completion(&vdev->destroy);  in vop_open()
884 	complete(&vdev->destroy);  in vop_open()
890 	struct vop_vdev *vdev = f->private_data, *vdev_tmp;  in vop_release()  local
891 	struct vop_info *vi = vdev->vi;  in vop_release()
895 	mutex_lock(&vdev->vdev_mutex);  in vop_release()
896 	if (vdev->deleted)  in vop_release()
901 		if (vdev == vdev_tmp) {  in vop_release()
902 			vop_virtio_del_device(vdev);  in vop_release()
910 	mutex_unlock(&vdev->vdev_mutex);  in vop_release()
912 		wait_for_completion(&vdev->destroy);  in vop_release()
914 	kfree(vdev);  in vop_release()
920 	struct vop_vdev *vdev = f->private_data;  in vop_ioctl()  local
921 	struct vop_info *vi = vdev->vi;  in vop_ioctl()
946 		mutex_lock(&vdev->vdev_mutex);  in vop_ioctl()
948 		ret = vop_virtio_add_device(vdev, dd_config);  in vop_ioctl()
951 		list_add_tail(&vdev->list, &vi->vdev_list);  in vop_ioctl()
954 		mutex_unlock(&vdev->vdev_mutex);  in vop_ioctl()
963 		mutex_lock(&vdev->vdev_mutex);  in vop_ioctl()
964 		ret = vop_vdev_inited(vdev);  in vop_ioctl()
973 		ret = vop_virtio_copy_desc(vdev, &copy);  in vop_ioctl()
981 		mutex_unlock(&vdev->vdev_mutex);  in vop_ioctl()
988 		mutex_lock(&vdev->vdev_mutex);  in vop_ioctl()
989 		ret = vop_vdev_inited(vdev);  in vop_ioctl()
992 		buf = memdup_user(argp, vdev->dd->config_len);  in vop_ioctl()
997 		ret = vop_virtio_config_change(vdev, buf);  in vop_ioctl()
1000 		mutex_unlock(&vdev->vdev_mutex);  in vop_ioctl()
1017 	struct vop_vdev *vdev = f->private_data;  in vop_poll()  local
1020 	mutex_lock(&vdev->vdev_mutex);  in vop_poll()
1021 	if (vop_vdev_inited(vdev)) {  in vop_poll()
1025 	poll_wait(f, &vdev->waitq, wait);  in vop_poll()
1026 	if (vop_vdev_inited(vdev)) {  in vop_poll()
1028 	} else if (vdev->poll_wake) {  in vop_poll()
1029 		vdev->poll_wake = 0;  in vop_poll()
1033 	mutex_unlock(&vdev->vdev_mutex);  in vop_poll()
1038 vop_query_offset(struct vop_vdev *vdev, unsigned long offset,  in vop_query_offset()  argument
1041 	struct vop_device *vpdev = vdev->vpdev;  in vop_query_offset()
1059 	for (i = 0; i < vdev->dd->num_vq; i++) {  in vop_query_offset()
1060 		struct vop_vringh *vvr = &vdev->vvr[i];  in vop_query_offset()
1077 	struct vop_vdev *vdev = f->private_data;  in vop_mmap()  local
1082 	err = vop_vdev_inited(vdev);  in vop_mmap()
1090 		i = vop_query_offset(vdev, offset, &size, &pa);  in vop_mmap()
1140 	struct vop_vdev *vdev;  in vop_host_uninit()  local
1145 		vdev = list_entry(pos, struct vop_vdev, list);  in vop_host_uninit()
1147 		reinit_completion(&vdev->destroy);  in vop_host_uninit()
1149 		mutex_lock(&vdev->vdev_mutex);  in vop_host_uninit()
1150 		vop_virtio_del_device(vdev);  in vop_host_uninit()
1151 		vdev->deleted = true;  in vop_host_uninit()
1152 		mutex_unlock(&vdev->vdev_mutex);  in vop_host_uninit()
1153 		complete(&vdev->destroy);  in vop_host_uninit()