87 static bool vfio_iommu_driver_allowed(struct vfio_container *container,  in vfio_iommu_driver_allowed()  argument
92 	return container->noiommu == (driver->ops == &vfio_noiommu_ops);  in vfio_iommu_driver_allowed()
155 	struct vfio_container *container;  in vfio_container_release()  local
156 	container = container_of(kref, struct vfio_container, kref);  in vfio_container_release()
158 	kfree(container);  in vfio_container_release()
161 static void vfio_container_get(struct vfio_container *container)  in vfio_container_get()  argument
163 	kref_get(&container->kref);  in vfio_container_get()
166 static void vfio_container_put(struct vfio_container *container)  in vfio_container_put()  argument
168 	kref_put(&container->kref, vfio_container_release);  in vfio_container_put()
174 		device->group->container->iommu_driver;  in vfio_device_container_register()
178 			device->group->container->iommu_data, device);  in vfio_device_container_register()
184 		device->group->container->iommu_driver;  in vfio_device_container_unregister()
188 			device->group->container->iommu_data, device);  in vfio_device_container_unregister()
191 long vfio_container_ioctl_check_extension(struct vfio_container *container,  in vfio_container_ioctl_check_extension()  argument
197 	down_read(&container->group_lock);  in vfio_container_ioctl_check_extension()
199 	driver = container->iommu_driver;  in vfio_container_ioctl_check_extension()
215 				if (!list_empty(&container->group_list) &&  in vfio_container_ioctl_check_extension()
216 				    !vfio_iommu_driver_allowed(container,  in vfio_container_ioctl_check_extension()
231 			ret = driver->ops->ioctl(container->iommu_data,  in vfio_container_ioctl_check_extension()
235 	up_read(&container->group_lock);  in vfio_container_ioctl_check_extension()
241 static int __vfio_container_attach_groups(struct vfio_container *container,  in __vfio_container_attach_groups()  argument
248 	list_for_each_entry(group, &container->group_list, container_next) {  in __vfio_container_attach_groups()
258 	list_for_each_entry_continue_reverse(group, &container->group_list,  in __vfio_container_attach_groups()
266 static long vfio_ioctl_set_iommu(struct vfio_container *container,  in vfio_ioctl_set_iommu()  argument
272 	down_write(&container->group_lock);  in vfio_ioctl_set_iommu()
282 	if (list_empty(&container->group_list) || container->iommu_driver) {  in vfio_ioctl_set_iommu()
283 		up_write(&container->group_lock);  in vfio_ioctl_set_iommu()
291 		if (!vfio_iommu_driver_allowed(container, driver))  in vfio_ioctl_set_iommu()
315 		ret = __vfio_container_attach_groups(container, driver, data);  in vfio_ioctl_set_iommu()
322 		container->iommu_driver = driver;  in vfio_ioctl_set_iommu()
323 		container->iommu_data = data;  in vfio_ioctl_set_iommu()
328 	up_write(&container->group_lock);  in vfio_ioctl_set_iommu()
336 	struct vfio_container *container = filep->private_data;  in vfio_fops_unl_ioctl()  local
341 	if (!container)  in vfio_fops_unl_ioctl()
349 		ret = vfio_container_ioctl_check_extension(container, arg);  in vfio_fops_unl_ioctl()
352 		ret = vfio_ioctl_set_iommu(container, arg);  in vfio_fops_unl_ioctl()
355 		driver = container->iommu_driver;  in vfio_fops_unl_ioctl()
356 		data = container->iommu_data;  in vfio_fops_unl_ioctl()
367 	struct vfio_container *container;  in vfio_fops_open()  local
369 	container = kzalloc(sizeof(*container), GFP_KERNEL);  in vfio_fops_open()
370 	if (!container)  in vfio_fops_open()
373 	INIT_LIST_HEAD(&container->group_list);  in vfio_fops_open()
374 	init_rwsem(&container->group_lock);  in vfio_fops_open()
375 	kref_init(&container->kref);  in vfio_fops_open()
377 	filep->private_data = container;  in vfio_fops_open()
384 	struct vfio_container *container = filep->private_data;  in vfio_fops_release()  local
385 	struct vfio_iommu_driver *driver = container->iommu_driver;  in vfio_fops_release()
388 		driver->ops->notify(container->iommu_data,  in vfio_fops_release()
393 	vfio_container_put(container);  in vfio_fops_release()
408 	struct vfio_container *container;  in vfio_container_from_file()  local
414 	container = file->private_data;  in vfio_container_from_file()
415 	WARN_ON(!container); /* fget ensures we don't race vfio_release */  in vfio_container_from_file()
416 	return container;  in vfio_container_from_file()
427 int vfio_container_attach_group(struct vfio_container *container,  in vfio_container_attach_group()  argument
438 	down_write(&container->group_lock);  in vfio_container_attach_group()
441 	if (!list_empty(&container->group_list) &&  in vfio_container_attach_group()
442 	    container->noiommu != (group->type == VFIO_NO_IOMMU)) {  in vfio_container_attach_group()
453 	driver = container->iommu_driver;  in vfio_container_attach_group()
455 		ret = driver->ops->attach_group(container->iommu_data,  in vfio_container_attach_group()
466 	group->container = container;  in vfio_container_attach_group()
468 	container->noiommu = (group->type == VFIO_NO_IOMMU);  in vfio_container_attach_group()
469 	list_add(&group->container_next, &container->group_list);  in vfio_container_attach_group()
472 	vfio_container_get(container);  in vfio_container_attach_group()
475 	up_write(&container->group_lock);  in vfio_container_attach_group()
481 	struct vfio_container *container = group->container;  in vfio_group_detach_container()  local
487 	down_write(&container->group_lock);  in vfio_group_detach_container()
489 	driver = container->iommu_driver;  in vfio_group_detach_container()
491 		driver->ops->detach_group(container->iommu_data,  in vfio_group_detach_container()
497 	group->container = NULL;  in vfio_group_detach_container()
502 	if (driver && list_empty(&container->group_list)) {  in vfio_group_detach_container()
503 		driver->ops->release(container->iommu_data);  in vfio_group_detach_container()
505 		container->iommu_driver = NULL;  in vfio_group_detach_container()
506 		container->iommu_data = NULL;  in vfio_group_detach_container()
509 	up_write(&container->group_lock);  in vfio_group_detach_container()
511 	vfio_container_put(container);  in vfio_group_detach_container()
520 	if (!group->container || !group->container->iommu_driver ||  in vfio_device_assign_container()
558 	struct vfio_container *container;  in vfio_pin_pages()  local
570 	container = group->container;  in vfio_pin_pages()
571 	driver = container->iommu_driver;  in vfio_pin_pages()
573 		ret = driver->ops->pin_pages(container->iommu_data,  in vfio_pin_pages()
592 	struct vfio_container *container;  in vfio_unpin_pages()  local
602 	container = device->group->container;  in vfio_unpin_pages()
603 	driver = container->iommu_driver;  in vfio_unpin_pages()
605 	driver->ops->unpin_pages(container->iommu_data, iova, npage);  in vfio_unpin_pages()
629 	struct vfio_container *container;  in vfio_dma_rw()  local
637 	container = device->group->container;  in vfio_dma_rw()
638 	driver = container->iommu_driver;  in vfio_dma_rw()
641 		ret = driver->ops->dma_rw(container->iommu_data,  in vfio_dma_rw()