61 	struct vhost_vdpa *v = container_of(vq->dev, struct vhost_vdpa, vdev);  in handle_vq_kick()  local
62 	const struct vdpa_config_ops *ops = v->vdpa->config;  in handle_vq_kick()
64 	ops->kick_vq(v->vdpa, vq - v->vqs);  in handle_vq_kick()
80 	struct vhost_vdpa *v = private;  in vhost_vdpa_config_cb()  local
81 	struct eventfd_ctx *config_ctx = v->config_ctx;  in vhost_vdpa_config_cb()
89 static void vhost_vdpa_setup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_setup_vq_irq()  argument
91 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_setup_vq_irq()
92 	const struct vdpa_config_ops *ops = v->vdpa->config;  in vhost_vdpa_setup_vq_irq()
93 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_setup_vq_irq()
101 	if (!vq->call_ctx.ctx || irq < 0)  in vhost_vdpa_setup_vq_irq()
108 		dev_info(&v->dev, "vq %u, irq bypass producer (token %p) registration fails, ret =  %d\n",  in vhost_vdpa_setup_vq_irq()
112 static void vhost_vdpa_unsetup_vq_irq(struct vhost_vdpa *v, u16 qid)  in vhost_vdpa_unsetup_vq_irq()  argument
114 	struct vhost_virtqueue *vq = &v->vqs[qid];  in vhost_vdpa_unsetup_vq_irq()
119 static void vhost_vdpa_reset(struct vhost_vdpa *v)  in vhost_vdpa_reset()  argument
121 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_reset()
124 	v->in_batch = 0;  in vhost_vdpa_reset()
127 static long vhost_vdpa_get_device_id(struct vhost_vdpa *v, u8 __user *argp)  in vhost_vdpa_get_device_id()  argument
129 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_device_id()
138 	return 0;  in vhost_vdpa_get_device_id()
141 static long vhost_vdpa_get_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_get_status()  argument
143 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_status()
152 	return 0;  in vhost_vdpa_get_status()
155 static long vhost_vdpa_set_status(struct vhost_vdpa *v, u8 __user *statusp)  in vhost_vdpa_set_status()  argument
157 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_status()
160 	int nvqs = v->nvqs;  in vhost_vdpa_set_status()
170 	 * status to 0.  in vhost_vdpa_set_status()
172 	if (status != 0 && (ops->get_status(vdpa) & ~status) != 0)  in vhost_vdpa_set_status()
178 		for (i = 0; i < nvqs; i++)  in vhost_vdpa_set_status()
179 			vhost_vdpa_setup_vq_irq(v, i);  in vhost_vdpa_set_status()
182 		for (i = 0; i < nvqs; i++)  in vhost_vdpa_set_status()
183 			vhost_vdpa_unsetup_vq_irq(v, i);  in vhost_vdpa_set_status()
185 	return 0;  in vhost_vdpa_set_status()
188 static int vhost_vdpa_config_validate(struct vhost_vdpa *v,  in vhost_vdpa_config_validate()  argument
191 	long size = 0;  in vhost_vdpa_config_validate()
193 	switch (v->virtio_id) {  in vhost_vdpa_config_validate()
199 	if (c->len == 0)  in vhost_vdpa_config_validate()
205 	return 0;  in vhost_vdpa_config_validate()
208 static long vhost_vdpa_get_config(struct vhost_vdpa *v,  in vhost_vdpa_get_config()  argument
211 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_config()
218 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_get_config()
232 	return 0;  in vhost_vdpa_get_config()
235 static long vhost_vdpa_set_config(struct vhost_vdpa *v,  in vhost_vdpa_set_config()  argument
238 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_config()
246 	if (vhost_vdpa_config_validate(v, &config))  in vhost_vdpa_set_config()
260 	return 0;  in vhost_vdpa_set_config()
263 static long vhost_vdpa_get_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_get_features()  argument
265 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_features()
274 	return 0;  in vhost_vdpa_get_features()
277 static long vhost_vdpa_set_features(struct vhost_vdpa *v, u64 __user *featurep)  in vhost_vdpa_set_features()  argument
279 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_features()
296 	return 0;  in vhost_vdpa_set_features()
299 static long vhost_vdpa_get_vring_num(struct vhost_vdpa *v, u16 __user *argp)  in vhost_vdpa_get_vring_num()  argument
301 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_get_vring_num()
310 	return 0;  in vhost_vdpa_get_vring_num()
313 static void vhost_vdpa_config_put(struct vhost_vdpa *v)  in vhost_vdpa_config_put()  argument
315 	if (v->config_ctx)  in vhost_vdpa_config_put()
316 		eventfd_ctx_put(v->config_ctx);  in vhost_vdpa_config_put()
319 static long vhost_vdpa_set_config_call(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_set_config_call()  argument
326 	cb.private = v->vdpa;  in vhost_vdpa_set_config_call()
331 	swap(ctx, v->config_ctx);  in vhost_vdpa_set_config_call()
336 	if (IS_ERR(v->config_ctx))  in vhost_vdpa_set_config_call()
337 		return PTR_ERR(v->config_ctx);  in vhost_vdpa_set_config_call()
339 	v->vdpa->config->set_config_cb(v->vdpa, &cb);  in vhost_vdpa_set_config_call()
341 	return 0;  in vhost_vdpa_set_config_call()
344 static long vhost_vdpa_get_iova_range(struct vhost_vdpa *v, u32 __user *argp)  in vhost_vdpa_get_iova_range()  argument
347 		.first = v->range.first,  in vhost_vdpa_get_iova_range()
348 		.last = v->range.last,  in vhost_vdpa_get_iova_range()
353 	return 0;  in vhost_vdpa_get_iova_range()
356 static long vhost_vdpa_vring_ioctl(struct vhost_vdpa *v, unsigned int cmd,  in vhost_vdpa_vring_ioctl()  argument
359 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_vring_ioctl()
369 	if (r < 0)  in vhost_vdpa_vring_ioctl()
372 	if (idx >= v->nvqs)  in vhost_vdpa_vring_ioctl()
375 	idx = array_index_nospec(idx, v->nvqs);  in vhost_vdpa_vring_ioctl()
376 	vq = &v->vqs[idx];  in vhost_vdpa_vring_ioctl()
383 		return 0;  in vhost_vdpa_vring_ioctl()
385 		r = ops->get_vq_state(v->vdpa, idx, &vq_state);  in vhost_vdpa_vring_ioctl()
393 	r = vhost_vring_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_vring_ioctl()
421 		vhost_vdpa_setup_vq_irq(v, idx);  in vhost_vdpa_vring_ioctl()
435 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_unlocked_ioctl()  local
436 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_unlocked_ioctl()
440 	long r = 0;  in vhost_vdpa_unlocked_ioctl()
447 		vhost_set_backend_features(&v->vdev, features);  in vhost_vdpa_unlocked_ioctl()
448 		return 0;  in vhost_vdpa_unlocked_ioctl()
455 		r = vhost_vdpa_get_device_id(v, argp);  in vhost_vdpa_unlocked_ioctl()
458 		r = vhost_vdpa_get_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
461 		r = vhost_vdpa_set_status(v, argp);  in vhost_vdpa_unlocked_ioctl()
464 		r = vhost_vdpa_get_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
467 		r = vhost_vdpa_set_config(v, argp);  in vhost_vdpa_unlocked_ioctl()
470 		r = vhost_vdpa_get_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
473 		r = vhost_vdpa_set_features(v, argp);  in vhost_vdpa_unlocked_ioctl()
476 		r = vhost_vdpa_get_vring_num(v, argp);  in vhost_vdpa_unlocked_ioctl()
483 		r = vhost_vdpa_set_config_call(v, argp);  in vhost_vdpa_unlocked_ioctl()
491 		r = vhost_vdpa_get_iova_range(v, argp);  in vhost_vdpa_unlocked_ioctl()
494 		r = vhost_dev_ioctl(&v->vdev, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
496 			r = vhost_vdpa_vring_ioctl(v, cmd, argp);  in vhost_vdpa_unlocked_ioctl()
504 static void vhost_vdpa_iotlb_unmap(struct vhost_vdpa *v, u64 start, u64 last)  in vhost_vdpa_iotlb_unmap()  argument
506 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_iotlb_unmap()
515 		     pinned > 0; pfn++, pinned--) {  in vhost_vdpa_iotlb_unmap()
526 static void vhost_vdpa_iotlb_free(struct vhost_vdpa *v)  in vhost_vdpa_iotlb_free()  argument
528 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_iotlb_free()
530 	vhost_vdpa_iotlb_unmap(v, 0ULL, 0ULL - 1);  in vhost_vdpa_iotlb_free()
537 	int flags = 0;  in perm_to_iommu_flags()
557 static int vhost_vdpa_map(struct vhost_vdpa *v,  in vhost_vdpa_map()  argument
560 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_map()
561 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_map()
563 	int r = 0;  in vhost_vdpa_map()
573 		if (!v->in_batch)  in vhost_vdpa_map()
576 		r = iommu_map(v->domain, iova, pa, size,  in vhost_vdpa_map()
588 static void vhost_vdpa_unmap(struct vhost_vdpa *v, u64 iova, u64 size)  in vhost_vdpa_unmap()  argument
590 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_unmap()
591 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_unmap()
594 	vhost_vdpa_iotlb_unmap(v, iova, iova + size - 1);  in vhost_vdpa_unmap()
599 		if (!v->in_batch)  in vhost_vdpa_unmap()
602 		iommu_unmap(v->domain, iova, size);  in vhost_vdpa_unmap()
606 static int vhost_vdpa_process_iotlb_update(struct vhost_vdpa *v,  in vhost_vdpa_process_iotlb_update()  argument
609 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_process_iotlb_update()
614 	unsigned long npages, cur_base, map_pfn, last_pfn = 0;  in vhost_vdpa_process_iotlb_update()
618 	int ret = 0;  in vhost_vdpa_process_iotlb_update()
620 	if (msg->iova < v->range.first ||  in vhost_vdpa_process_iotlb_update()
621 	    msg->iova + msg->size - 1 > v->range.last)  in vhost_vdpa_process_iotlb_update()
652 	nchunks = 0;  in vhost_vdpa_process_iotlb_update()
659 			if (pinned < 0) {  in vhost_vdpa_process_iotlb_update()
670 			map_pfn = page_to_pfn(page_list[0]);  in vhost_vdpa_process_iotlb_update()
672 		for (i = 0; i < pinned; i++) {  in vhost_vdpa_process_iotlb_update()
679 				ret = vhost_vdpa_map(v, iova, csize,  in vhost_vdpa_process_iotlb_update()
698 				nchunks = 0;  in vhost_vdpa_process_iotlb_update()
709 	ret = vhost_vdpa_map(v, iova, (last_pfn - map_pfn + 1) << PAGE_SHIFT,  in vhost_vdpa_process_iotlb_update()
729 		vhost_vdpa_unmap(v, msg->iova, msg->size);  in vhost_vdpa_process_iotlb_update()
741 	struct vhost_vdpa *v = container_of(dev, struct vhost_vdpa, vdev);  in vhost_vdpa_process_iotlb_msg()  local
742 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_process_iotlb_msg()
744 	int r = 0;  in vhost_vdpa_process_iotlb_msg()
752 		r = vhost_vdpa_process_iotlb_update(v, msg);  in vhost_vdpa_process_iotlb_msg()
755 		vhost_vdpa_unmap(v, msg->iova, msg->size);  in vhost_vdpa_process_iotlb_msg()
758 		v->in_batch = true;  in vhost_vdpa_process_iotlb_msg()
761 		if (v->in_batch && ops->set_map)  in vhost_vdpa_process_iotlb_msg()
763 		v->in_batch = false;  in vhost_vdpa_process_iotlb_msg()
777 	struct vhost_vdpa *v = file->private_data;  in vhost_vdpa_chr_write_iter()  local
778 	struct vhost_dev *dev = &v->vdev;  in vhost_vdpa_chr_write_iter()
783 static int vhost_vdpa_alloc_domain(struct vhost_vdpa *v)  in vhost_vdpa_alloc_domain()  argument
785 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_alloc_domain()
793 		return 0;  in vhost_vdpa_alloc_domain()
802 	v->domain = iommu_domain_alloc(bus);  in vhost_vdpa_alloc_domain()
803 	if (!v->domain)  in vhost_vdpa_alloc_domain()
806 	ret = iommu_attach_device(v->domain, dma_dev);  in vhost_vdpa_alloc_domain()
810 	return 0;  in vhost_vdpa_alloc_domain()
813 	iommu_domain_free(v->domain);  in vhost_vdpa_alloc_domain()
817 static void vhost_vdpa_free_domain(struct vhost_vdpa *v)  in vhost_vdpa_free_domain()  argument
819 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_free_domain()
822 	if (v->domain) {  in vhost_vdpa_free_domain()
823 		iommu_detach_device(v->domain, dma_dev);  in vhost_vdpa_free_domain()
824 		iommu_domain_free(v->domain);  in vhost_vdpa_free_domain()
827 	v->domain = NULL;  in vhost_vdpa_free_domain()
830 static void vhost_vdpa_set_iova_range(struct vhost_vdpa *v)  in vhost_vdpa_set_iova_range()  argument
832 	struct vdpa_iova_range *range = &v->range;  in vhost_vdpa_set_iova_range()
834 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_set_iova_range()
839 	} else if (v->domain &&  in vhost_vdpa_set_iova_range()
840 		   !iommu_domain_get_attr(v->domain,  in vhost_vdpa_set_iova_range()
846 		range->first = 0;  in vhost_vdpa_set_iova_range()
853 	struct vhost_vdpa *v;  in vhost_vdpa_open()  local
858 	v = container_of(inode->i_cdev, struct vhost_vdpa, cdev);  in vhost_vdpa_open()
860 	opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_open()
864 	nvqs = v->nvqs;  in vhost_vdpa_open()
865 	vhost_vdpa_reset(v);  in vhost_vdpa_open()
873 	dev = &v->vdev;  in vhost_vdpa_open()
874 	for (i = 0; i < nvqs; i++) {  in vhost_vdpa_open()
875 		vqs[i] = &v->vqs[i];  in vhost_vdpa_open()
878 	vhost_dev_init(dev, vqs, nvqs, 0, 0, 0, false,  in vhost_vdpa_open()
881 	dev->iotlb = vhost_iotlb_alloc(0, 0);  in vhost_vdpa_open()
887 	r = vhost_vdpa_alloc_domain(v);  in vhost_vdpa_open()
891 	vhost_vdpa_set_iova_range(v);  in vhost_vdpa_open()
893 	filep->private_data = v;  in vhost_vdpa_open()
895 	return 0;  in vhost_vdpa_open()
898 	vhost_dev_cleanup(&v->vdev);  in vhost_vdpa_open()
901 	atomic_dec(&v->opened);  in vhost_vdpa_open()
905 static void vhost_vdpa_clean_irq(struct vhost_vdpa *v)  in vhost_vdpa_clean_irq()  argument
910 	for (i = 0; i < v->nvqs; i++) {  in vhost_vdpa_clean_irq()
911 		vq = &v->vqs[i];  in vhost_vdpa_clean_irq()
919 	struct vhost_vdpa *v = filep->private_data;  in vhost_vdpa_release()  local
920 	struct vhost_dev *d = &v->vdev;  in vhost_vdpa_release()
924 	vhost_vdpa_reset(v);  in vhost_vdpa_release()
925 	vhost_dev_stop(&v->vdev);  in vhost_vdpa_release()
926 	vhost_vdpa_iotlb_free(v);  in vhost_vdpa_release()
927 	vhost_vdpa_free_domain(v);  in vhost_vdpa_release()
928 	vhost_vdpa_config_put(v);  in vhost_vdpa_release()
929 	vhost_vdpa_clean_irq(v);  in vhost_vdpa_release()
930 	vhost_dev_cleanup(&v->vdev);  in vhost_vdpa_release()
931 	kfree(v->vdev.vqs);  in vhost_vdpa_release()
934 	atomic_dec(&v->opened);  in vhost_vdpa_release()
935 	complete(&v->completion);  in vhost_vdpa_release()
937 	return 0;  in vhost_vdpa_release()
943 	struct vhost_vdpa *v = vmf->vma->vm_file->private_data;  in vhost_vdpa_fault()  local
944 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_fault()
967 	struct vhost_vdpa *v = vma->vm_file->private_data;  in vhost_vdpa_mmap()  local
968 	struct vdpa_device *vdpa = v->vdpa;  in vhost_vdpa_mmap()
975 	if ((vma->vm_flags & VM_SHARED) == 0)  in vhost_vdpa_mmap()
995 	return 0;  in vhost_vdpa_mmap()
1013 	struct vhost_vdpa *v =  in vhost_vdpa_release_dev()  local
1016 	ida_simple_remove(&vhost_vdpa_ida, v->minor);  in vhost_vdpa_release_dev()
1017 	kfree(v->vqs);  in vhost_vdpa_release_dev()
1018 	kfree(v);  in vhost_vdpa_release_dev()
1024 	struct vhost_vdpa *v;  in vhost_vdpa_probe()  local
1032 	v = kzalloc(sizeof(*v), GFP_KERNEL | __GFP_RETRY_MAYFAIL);  in vhost_vdpa_probe()
1033 	if (!v)  in vhost_vdpa_probe()
1036 	minor = ida_simple_get(&vhost_vdpa_ida, 0,  in vhost_vdpa_probe()
1038 	if (minor < 0) {  in vhost_vdpa_probe()
1039 		kfree(v);  in vhost_vdpa_probe()
1043 	atomic_set(&v->opened, 0);  in vhost_vdpa_probe()
1044 	v->minor = minor;  in vhost_vdpa_probe()
1045 	v->vdpa = vdpa;  in vhost_vdpa_probe()
1046 	v->nvqs = vdpa->nvqs;  in vhost_vdpa_probe()
1047 	v->virtio_id = ops->get_device_id(vdpa);  in vhost_vdpa_probe()
1049 	device_initialize(&v->dev);  in vhost_vdpa_probe()
1050 	v->dev.release = vhost_vdpa_release_dev;  in vhost_vdpa_probe()
1051 	v->dev.parent = &vdpa->dev;  in vhost_vdpa_probe()
1052 	v->dev.devt = MKDEV(MAJOR(vhost_vdpa_major), minor);  in vhost_vdpa_probe()
1053 	v->vqs = kmalloc_array(v->nvqs, sizeof(struct vhost_virtqueue),  in vhost_vdpa_probe()
1055 	if (!v->vqs) {  in vhost_vdpa_probe()
1060 	r = dev_set_name(&v->dev, "vhost-vdpa-%u", minor);  in vhost_vdpa_probe()
1064 	cdev_init(&v->cdev, &vhost_vdpa_fops);  in vhost_vdpa_probe()
1065 	v->cdev.owner = THIS_MODULE;  in vhost_vdpa_probe()
1067 	r = cdev_device_add(&v->cdev, &v->dev);  in vhost_vdpa_probe()
1071 	init_completion(&v->completion);  in vhost_vdpa_probe()
1072 	vdpa_set_drvdata(vdpa, v);  in vhost_vdpa_probe()
1074 	return 0;  in vhost_vdpa_probe()
1077 	put_device(&v->dev);  in vhost_vdpa_probe()
1083 	struct vhost_vdpa *v = vdpa_get_drvdata(vdpa);  in vhost_vdpa_remove()  local
1086 	cdev_device_del(&v->cdev, &v->dev);  in vhost_vdpa_remove()
1089 		opened = atomic_cmpxchg(&v->opened, 0, 1);  in vhost_vdpa_remove()
1092 		wait_for_completion(&v->completion);  in vhost_vdpa_remove()
1095 	put_device(&v->dev);  in vhost_vdpa_remove()
1110 	r = alloc_chrdev_region(&vhost_vdpa_major, 0, VHOST_VDPA_DEV_MAX,  in vhost_vdpa_init()
1119 	return 0;  in vhost_vdpa_init()