61 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);  in handle_vq_kick()  local
62 	const struct vdpa_config_ops *ops = v->vdpa->config;  in handle_vq_kick()
64 	ops->kick_vq(v->vdpa, vq - v->vqs);  in handle_vq_kick()
80 	struct vhost_vdpa *v = private;  in vhost_vdpa_config_cb()  local
81 	struct eventfd_ctx *config_ctx = v->config_ctx;  in vhost_vdpa_config_cb()
89 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_setup_vq_irq()  argument
91 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_setup_vq_irq()
92 	const struct vdpa_config_ops *ops = v->vdpa->config;  in vhost_vdpa_setup_vq_irq()
93 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_setup_vq_irq()
101 	if (!vq->call_ctx.ctx || irq < 0)  in vhost_vdpa_setup_vq_irq()
108 		dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",  in vhost_vdpa_setup_vq_irq()
112 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_unsetup_vq_irq()  argument
114 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_unsetup_vq_irq()
119 static int vhost_vdpa_reset(struct vhost_vdpa *v)  in vhost_vdpa_reset()  argument
121 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_reset()
123 	v->in_batch = 0;  in vhost_vdpa_reset()
128 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)  in vhost_vdpa_get_device_id()  argument
130 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_device_id()
139 	return 0;  in vhost_vdpa_get_device_id()
142 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_get_status()  argument
144 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_status()
153 	return 0;  in vhost_vdpa_get_status()
156 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_set_status()  argument
158 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_status()
161 	int ret, nvqs = v->nvqs;  in vhost_vdpa_set_status()
171 	 * status to 0.  in vhost_vdpa_set_status()
173 	if (status != 0 && (ops->get_status(vdpa) & ~status) != 0)  in vhost_vdpa_set_status()
177 		for (i = 0; i < nvqs; i++)  in vhost_vdpa_set_status()
178 			vhost_vdpa_unsetup_vq_irq(v, i);  in vhost_vdpa_set_status()
180 	if (status == 0) {  in vhost_vdpa_set_status()
188 		for (i = 0; i < nvqs; i++)  in vhost_vdpa_set_status()
189 			vhost_vdpa_setup_vq_irq(v, i);  in vhost_vdpa_set_status()
191 	return 0;  in vhost_vdpa_set_status()
194 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,  in vhost_vdpa_config_validate()  argument
197 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_config_validate()
200 	if (c->len == 0)  in vhost_vdpa_config_validate()
206 	return 0;  in vhost_vdpa_config_validate()
209 static long vhost_vdpa_get_config(struct vhost_vdpa *v,  in vhost_vdpa_get_config()  argument
212 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_config()
219 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_get_config()
233 	return 0;  in vhost_vdpa_get_config()
236 static long vhost_vdpa_set_config(struct vhost_vdpa *v,  in vhost_vdpa_set_config()  argument
239 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_config()
247 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_set_config()
257 	return 0;  in vhost_vdpa_set_config()
260 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_get_features()  argument
262 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_features()
271 	return 0;  in vhost_vdpa_get_features()
274 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_set_features()  argument
276 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_features()
293 	return 0;  in vhost_vdpa_set_features()
296 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)  in vhost_vdpa_get_vring_num()  argument
298 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_vring_num()
307 	return 0;  in vhost_vdpa_get_vring_num()
310 static void vhost_vdpa_config_put(struct vhost_vdpa *v)  in vhost_vdpa_config_put()  argument
312 	if (v->config_ctx) {  in vhost_vdpa_config_put()
313 		eventfd_ctx_put(v->config_ctx);  in vhost_vdpa_config_put()
314 		v->config_ctx = NULL;  in vhost_vdpa_config_put()
318 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_set_config_call()  argument
325 	cb.private = v;  in vhost_vdpa_set_config_call()
330 	swap(ctx, v->config_ctx);  in vhost_vdpa_set_config_call()
335 	if (IS_ERR(v->config_ctx)) {  in vhost_vdpa_set_config_call()
336 		long ret = PTR_ERR(v->config_ctx);  in vhost_vdpa_set_config_call()
338 		v->config_ctx = NULL;  in vhost_vdpa_set_config_call()
342 	v->vdpa->config->set_config_cb(v->vdpa, &cb);  in vhost_vdpa_set_config_call()
344 	return 0;  in vhost_vdpa_set_config_call()
347 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_get_iova_range()  argument
350 		.first = v->range.first,  in vhost_vdpa_get_iova_range()
351 		.last = v->range.last,  in vhost_vdpa_get_iova_range()
356 	return 0;  in vhost_vdpa_get_iova_range()
359 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,  in vhost_vdpa_vring_ioctl()  argument
362 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_vring_ioctl()
372 	if (r < 0)  in vhost_vdpa_vring_ioctl()
375 	if (idx >= v->nvqs)  in vhost_vdpa_vring_ioctl()
378 	idx = array_index_nospec(idx, v->nvqs);  in vhost_vdpa_vring_ioctl()
379 	vq = &v->vqs[idx];  in vhost_vdpa_vring_ioctl()
386 		return 0;  in vhost_vdpa_vring_ioctl()
388 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);  in vhost_vdpa_vring_ioctl()
396 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_vring_ioctl()
424 		vhost_vdpa_setup_vq_irq(v, idx);  in vhost_vdpa_vring_ioctl()
438 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_unlocked_ioctl()  local
439 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_unlocked_ioctl()
443 	long r = 0;  in vhost_vdpa_unlocked_ioctl()
450 		vhost_set_backend_features(&v->vdev, features);  in vhost_vdpa_unlocked_ioctl()
451 		return 0;  in vhost_vdpa_unlocked_ioctl()
458 		r = vhost_vdpa_get_device_id(v, argp);  in vhost_vdpa_unlocked_ioctl()
461 		r = vhost_vdpa_get_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
464 		r = vhost_vdpa_set_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
467 		r = vhost_vdpa_get_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
470 		r = vhost_vdpa_set_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
473 		r = vhost_vdpa_get_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
476 		r = vhost_vdpa_set_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
479 		r = vhost_vdpa_get_vring_num(v, argp);  in vhost_vdpa_unlocked_ioctl()
486 		r = vhost_vdpa_set_config_call(v, argp);  in vhost_vdpa_unlocked_ioctl()
494 		r = vhost_vdpa_get_iova_range(v, argp);  in vhost_vdpa_unlocked_ioctl()
497 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
499 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
507 static void vhost_vdpa_pa_unmap(struct vhost_vdpa *v, u64 start, u64 last)  in vhost_vdpa_pa_unmap()  argument
509 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_pa_unmap()
518 		     pinned > 0; pfn++, pinned--) {  in vhost_vdpa_pa_unmap()
529 static void vhost_vdpa_va_unmap(struct vhost_vdpa *v, u64 start, u64 last)  in vhost_vdpa_va_unmap()  argument
531 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_va_unmap()
544 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last)  in vhost_vdpa_iotlb_unmap()  argument
546 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_iotlb_unmap()
549 		return vhost_vdpa_va_unmap(v, start, last);  in vhost_vdpa_iotlb_unmap()
551 	return vhost_vdpa_pa_unmap(v, start, last);  in vhost_vdpa_iotlb_unmap()
554 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v)  in vhost_vdpa_iotlb_free()  argument
556 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_iotlb_free()
558 	vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1);  in vhost_vdpa_iotlb_free()
565 	int flags = 0;  in perm_to_iommu_flags()
585 static int vhost_vdpa_map(struct vhost_vdpa *v, u64 iova,  in vhost_vdpa_map()  argument
588 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_map()
589 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_map()
591 	int r = 0;  in vhost_vdpa_map()
601 		if (!v->in_batch)  in vhost_vdpa_map()
604 		r = iommu_map(v->domain, iova, pa, size,  in vhost_vdpa_map()
615 	return 0;  in vhost_vdpa_map()
618 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)  in vhost_vdpa_unmap()  argument
620 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_unmap()
621 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_unmap()
624 	vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);  in vhost_vdpa_unmap()
629 		if (!v->in_batch)  in vhost_vdpa_unmap()
632 		iommu_unmap(v->domain, iova, size);  in vhost_vdpa_unmap()
636 static int vhost_vdpa_va_map(struct vhost_vdpa *v,  in vhost_vdpa_va_map()  argument
639 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_va_map()
643 	int ret = 0;  in vhost_vdpa_va_map()
666 		ret = vhost_vdpa_map(v, map_iova, map_size, uaddr,  in vhost_vdpa_va_map()
679 		vhost_vdpa_unmap(v, iova, map_iova - iova);  in vhost_vdpa_va_map()
686 static int vhost_vdpa_pa_map(struct vhost_vdpa *v,  in vhost_vdpa_pa_map()  argument
689 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_pa_map()
693 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;  in vhost_vdpa_pa_map()
697 	int ret = 0;  in vhost_vdpa_pa_map()
723 	nchunks = 0;  in vhost_vdpa_pa_map()
730 			if (pinned < 0) {  in vhost_vdpa_pa_map()
741 			map_pfn = page_to_pfn(page_list[0]);  in vhost_vdpa_pa_map()
743 		for (i = 0; i < pinned; i++) {  in vhost_vdpa_pa_map()
750 				ret = vhost_vdpa_map(v, iova, csize,  in vhost_vdpa_pa_map()
769 				nchunks = 0;  in vhost_vdpa_pa_map()
780 	ret = vhost_vdpa_map(v, iova, PFN_PHYS(last_pfn - map_pfn + 1),  in vhost_vdpa_pa_map()
800 		vhost_vdpa_unmap(v, start, size);  in vhost_vdpa_pa_map()
810 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,  in vhost_vdpa_process_iotlb_update()  argument
813 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_process_iotlb_update()
814 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_process_iotlb_update()
817 	if (msg->iova < v->range.first || !msg->size ||  in vhost_vdpa_process_iotlb_update()
819 	    msg->iova + msg->size - 1 > v->range.last)  in vhost_vdpa_process_iotlb_update()
827 		return vhost_vdpa_va_map(v, msg->iova, msg->size,  in vhost_vdpa_process_iotlb_update()
830 	return vhost_vdpa_pa_map(v, msg->iova, msg->size, msg->uaddr,  in vhost_vdpa_process_iotlb_update()
837 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);  in vhost_vdpa_process_iotlb_msg()  local
838 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_process_iotlb_msg()
840 	int r = 0;  in vhost_vdpa_process_iotlb_msg()
850 		r = vhost_vdpa_process_iotlb_update(v, msg);  in vhost_vdpa_process_iotlb_msg()
853 		vhost_vdpa_unmap(v, msg->iova, msg->size);  in vhost_vdpa_process_iotlb_msg()
856 		v->in_batch = true;  in vhost_vdpa_process_iotlb_msg()
859 		if (v->in_batch && ops->set_map)  in vhost_vdpa_process_iotlb_msg()
861 		v->in_batch = false;  in vhost_vdpa_process_iotlb_msg()
877 	struct vhost_vdpa *v = file->private_data;  in vhost_vdpa_chr_write_iter()  local
878 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_chr_write_iter()
883 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)  in vhost_vdpa_alloc_domain()  argument
885 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_alloc_domain()
893 		return 0;  in vhost_vdpa_alloc_domain()
902 	v->domain = iommu_domain_alloc(bus);  in vhost_vdpa_alloc_domain()
903 	if (!v->domain)  in vhost_vdpa_alloc_domain()
906 	ret = iommu_attach_device(v->domain, dma_dev);  in vhost_vdpa_alloc_domain()
910 	return 0;  in vhost_vdpa_alloc_domain()
913 	iommu_domain_free(v->domain);  in vhost_vdpa_alloc_domain()
917 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)  in vhost_vdpa_free_domain()  argument
919 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_free_domain()
922 	if (v->domain) {  in vhost_vdpa_free_domain()
923 		iommu_detach_device(v->domain, dma_dev);  in vhost_vdpa_free_domain()
924 		iommu_domain_free(v->domain);  in vhost_vdpa_free_domain()
927 	v->domain = NULL;  in vhost_vdpa_free_domain()
930 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)  in vhost_vdpa_set_iova_range()  argument
932 	struct vdpa_iova_range *range = &v->range;  in vhost_vdpa_set_iova_range()
933 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_iova_range()
938 	} else if (v->domain && v->domain->geometry.force_aperture) {  in vhost_vdpa_set_iova_range()
939 		range->first = v->domain->geometry.aperture_start;  in vhost_vdpa_set_iova_range()
940 		range->last = v->domain->geometry.aperture_end;  in vhost_vdpa_set_iova_range()
942 		range->first = 0;  in vhost_vdpa_set_iova_range()
949 	struct vhost_vdpa *v;  in vhost_vdpa_open()  local
954 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);  in vhost_vdpa_open()
956 	opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_open()
960 	nvqs = v->nvqs;  in vhost_vdpa_open()
961 	r = vhost_vdpa_reset(v);  in vhost_vdpa_open()
971 	dev = &v->vdev;  in vhost_vdpa_open()
972 	for (i = 0; i < nvqs; i++) {  in vhost_vdpa_open()
973 		vqs[i] = &v->vqs[i];  in vhost_vdpa_open()
976 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,  in vhost_vdpa_open()
979 	dev->iotlb = vhost_iotlb_alloc(0, 0);  in vhost_vdpa_open()
985 	r = vhost_vdpa_alloc_domain(v);  in vhost_vdpa_open()
989 	vhost_vdpa_set_iova_range(v);  in vhost_vdpa_open()
991 	filep->private_data = v;  in vhost_vdpa_open()
993 	return 0;  in vhost_vdpa_open()
996 	vhost_dev_cleanup(&v->vdev);  in vhost_vdpa_open()
999 	atomic_dec(&v->opened);  in vhost_vdpa_open()
1003 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)  in vhost_vdpa_clean_irq()  argument
1007 	for (i = 0; i < v->nvqs; i++)  in vhost_vdpa_clean_irq()
1008 		vhost_vdpa_unsetup_vq_irq(v, i);  in vhost_vdpa_clean_irq()
1013 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_release()  local
1014 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_release()
1018 	vhost_vdpa_reset(v);  in vhost_vdpa_release()
1019 	vhost_dev_stop(&v->vdev);  in vhost_vdpa_release()
1020 	vhost_vdpa_iotlb_free(v);  in vhost_vdpa_release()
1021 	vhost_vdpa_free_domain(v);  in vhost_vdpa_release()
1022 	vhost_vdpa_config_put(v);  in vhost_vdpa_release()
1023 	vhost_vdpa_clean_irq(v);  in vhost_vdpa_release()
1024 	vhost_dev_cleanup(&v->vdev);  in vhost_vdpa_release()
1025 	kfree(v->vdev.vqs);  in vhost_vdpa_release()
1028 	atomic_dec(&v->opened);  in vhost_vdpa_release()
1029 	complete(&v->completion);  in vhost_vdpa_release()
1031 	return 0;  in vhost_vdpa_release()
1037 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;  in vhost_vdpa_fault()  local
1038 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_fault()
1061 	struct vhost_vdpa *v = vma->vm_file->private_data;  in vhost_vdpa_mmap()  local
1062 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_mmap()
1069 	if ((vma->vm_flags & VM_SHARED) == 0)  in vhost_vdpa_mmap()
1090 	return 0;  in vhost_vdpa_mmap()
1108 	struct vhost_vdpa *v =  in vhost_vdpa_release_dev()  local
1111 	ida_simple_remove(&vhost_vdpa_ida, v->minor);  in vhost_vdpa_release_dev()
1112 	kfree(v->vqs);  in vhost_vdpa_release_dev()
1113 	kfree(v);  in vhost_vdpa_release_dev()
1119 	struct vhost_vdpa *v;  in vhost_vdpa_probe()  local
1123 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);  in vhost_vdpa_probe()
1124 	if (!v)  in vhost_vdpa_probe()
1127 	minor = ida_simple_get(&vhost_vdpa_ida, 0,  in vhost_vdpa_probe()
1129 	if (minor < 0) {  in vhost_vdpa_probe()
1130 		kfree(v);  in vhost_vdpa_probe()
1134 	atomic_set(&v->opened, 0);  in vhost_vdpa_probe()
1135 	v->minor = minor;  in vhost_vdpa_probe()
1136 	v->vdpa = vdpa;  in vhost_vdpa_probe()
1137 	v->nvqs = vdpa->nvqs;  in vhost_vdpa_probe()
1138 	v->virtio_id = ops->get_device_id(vdpa);  in vhost_vdpa_probe()
1140 	device_initialize(&v->dev);  in vhost_vdpa_probe()
1141 	v->dev.release = vhost_vdpa_release_dev;  in vhost_vdpa_probe()
1142 	v->dev.parent = &vdpa->dev;  in vhost_vdpa_probe()
1143 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);  in vhost_vdpa_probe()
1144 	v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),  in vhost_vdpa_probe()
1146 	if (!v->vqs) {  in vhost_vdpa_probe()
1151 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);  in vhost_vdpa_probe()
1155 	cdev_init(&v->cdev, &vhost_vdpa_fops);  in vhost_vdpa_probe()
1156 	v->cdev.owner = THIS_MODULE;  in vhost_vdpa_probe()
1158 	r = cdev_device_add(&v->cdev, &v->dev);  in vhost_vdpa_probe()
1162 	init_completion(&v->completion);  in vhost_vdpa_probe()
1163 	vdpa_set_drvdata(vdpa, v);  in vhost_vdpa_probe()
1165 	return 0;  in vhost_vdpa_probe()
1168 	put_device(&v->dev);  in vhost_vdpa_probe()
1174 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);  in vhost_vdpa_remove()  local
1177 	cdev_device_del(&v->cdev, &v->dev);  in vhost_vdpa_remove()
1180 		opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_remove()
1183 		wait_for_completion(&v->completion);  in vhost_vdpa_remove()
1186 	put_device(&v->dev);  in vhost_vdpa_remove()
1201 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,  in vhost_vdpa_init()
1210 	return 0;  in vhost_vdpa_init()