75 #define for_each_group_device(group, pos) \  argument
76 	list_for_each_entry(pos, &(group)->devices, list)
80 	ssize_t (*show)(struct iommu_group *group, char *buf);
81 	ssize_t (*store)(struct iommu_group *group,
104 				struct iommu_group *group);
110 static int __iommu_device_set_domain(struct iommu_group *group,
114 static int __iommu_group_set_domain_internal(struct iommu_group *group,
117 static int __iommu_group_set_domain(struct iommu_group *group,  in __iommu_group_set_domain()  argument
120 	return __iommu_group_set_domain_internal(group, new_domain, 0);  in __iommu_group_set_domain()
122 static void __iommu_group_set_domain_nofail(struct iommu_group *group,  in __iommu_group_set_domain_nofail()  argument
126 		group, new_domain, IOMMU_SET_DOMAIN_MUST_SUCCEED));  in __iommu_group_set_domain_nofail()
129 static int iommu_setup_default_domain(struct iommu_group *group,
133 static ssize_t iommu_group_store_type(struct iommu_group *group,
135 static struct group_device *iommu_group_alloc_device(struct iommu_group *group,
137 static void __iommu_group_free_device(struct iommu_group *group,
391 	struct iommu_group *group;  in iommu_init_device()  local
412 	group = ops->device_group(dev);  in iommu_init_device()
413 	if (WARN_ON_ONCE(group == NULL))  in iommu_init_device()
414 		group = ERR_PTR(-EINVAL);  in iommu_init_device()
415 	if (IS_ERR(group)) {  in iommu_init_device()
416 		ret = PTR_ERR(group);  in iommu_init_device()
419 	dev->iommu_group = group;  in iommu_init_device()
441 	struct iommu_group *group = dev->iommu_group;  in iommu_deinit_device()  local
444 	lockdep_assert_held(&group->mutex);  in iommu_deinit_device()
464 	if (list_empty(&group->devices)) {  in iommu_deinit_device()
465 		if (group->default_domain) {  in iommu_deinit_device()
466 			iommu_domain_free(group->default_domain);  in iommu_deinit_device()
467 			group->default_domain = NULL;  in iommu_deinit_device()
469 		if (group->blocking_domain) {  in iommu_deinit_device()
470 			iommu_domain_free(group->blocking_domain);  in iommu_deinit_device()
471 			group->blocking_domain = NULL;  in iommu_deinit_device()
473 		group->domain = NULL;  in iommu_deinit_device()
485 	struct iommu_group *group;  in __iommu_probe_device()  local
511 	group = dev->iommu_group;  in __iommu_probe_device()
512 	gdev = iommu_group_alloc_device(group, dev);  in __iommu_probe_device()
513 	mutex_lock(&group->mutex);  in __iommu_probe_device()
523 	list_add_tail(&gdev->list, &group->devices);  in __iommu_probe_device()
524 	WARN_ON(group->default_domain && !group->domain);  in __iommu_probe_device()
525 	if (group->default_domain)  in __iommu_probe_device()
526 		iommu_create_device_direct_mappings(group->default_domain, dev);  in __iommu_probe_device()
527 	if (group->domain) {  in __iommu_probe_device()
528 		ret = __iommu_device_set_domain(group, dev, group->domain, 0);  in __iommu_probe_device()
531 	} else if (!group->default_domain && !group_list) {  in __iommu_probe_device()
532 		ret = iommu_setup_default_domain(group, 0);  in __iommu_probe_device()
535 	} else if (!group->default_domain) {  in __iommu_probe_device()
541 		if (list_empty(&group->entry))  in __iommu_probe_device()
542 			list_add_tail(&group->entry, group_list);  in __iommu_probe_device()
544 	mutex_unlock(&group->mutex);  in __iommu_probe_device()
554 	__iommu_group_free_device(group, gdev);  in __iommu_probe_device()
557 	mutex_unlock(&group->mutex);  in __iommu_probe_device()
558 	iommu_group_put(group);  in __iommu_probe_device()
581 static void __iommu_group_free_device(struct iommu_group *group,  in __iommu_group_free_device()  argument
586 	sysfs_remove_link(group->devices_kobj, grp_dev->name);  in __iommu_group_free_device()
589 	trace_remove_device_from_group(group->id, dev);  in __iommu_group_free_device()
596 	if (list_empty(&group->devices))  in __iommu_group_free_device()
597 		WARN_ON(group->owner_cnt ||  in __iommu_group_free_device()
598 			group->domain != group->default_domain);  in __iommu_group_free_device()
607 	struct iommu_group *group = dev->iommu_group;  in __iommu_group_remove_device()  local
610 	mutex_lock(&group->mutex);  in __iommu_group_remove_device()
611 	for_each_group_device(group, device) {  in __iommu_group_remove_device()
616 		__iommu_group_free_device(group, device);  in __iommu_group_remove_device()
623 	mutex_unlock(&group->mutex);  in __iommu_group_remove_device()
629 	iommu_group_put(group);  in __iommu_group_remove_device()
634 	struct iommu_group *group = dev->iommu_group;  in iommu_release_device()  local
636 	if (group)  in iommu_release_device()
683 	struct iommu_group *group = to_iommu_group(kobj);  in iommu_group_attr_show()  local
687 		ret = attr->show(group, buf);  in iommu_group_attr_show()
696 	struct iommu_group *group = to_iommu_group(kobj);  in iommu_group_attr_store()  local
700 		ret = attr->store(group, buf, count);  in iommu_group_attr_store()
709 static int iommu_group_create_file(struct iommu_group *group,  in iommu_group_create_file()  argument
712 	return sysfs_create_file(&group->kobj, &attr->attr);  in iommu_group_create_file()
715 static void iommu_group_remove_file(struct iommu_group *group,  in iommu_group_remove_file()  argument
718 	sysfs_remove_file(&group->kobj, &attr->attr);  in iommu_group_remove_file()
721 static ssize_t iommu_group_show_name(struct iommu_group *group, char *buf)  in iommu_group_show_name()  argument
723 	return sysfs_emit(buf, "%s\n", group->name);  in iommu_group_show_name()
802 int iommu_get_group_resv_regions(struct iommu_group *group,  in iommu_get_group_resv_regions()  argument
808 	mutex_lock(&group->mutex);  in iommu_get_group_resv_regions()
809 	for_each_group_device(group, device) {  in iommu_get_group_resv_regions()
826 	mutex_unlock(&group->mutex);  in iommu_get_group_resv_regions()
831 static ssize_t iommu_group_show_resv_regions(struct iommu_group *group,  in iommu_group_show_resv_regions()  argument
839 	iommu_get_group_resv_regions(group, &group_resv_regions);  in iommu_group_show_resv_regions()
853 static ssize_t iommu_group_show_type(struct iommu_group *group,  in iommu_group_show_type()  argument
858 	mutex_lock(&group->mutex);  in iommu_group_show_type()
859 	if (group->default_domain) {  in iommu_group_show_type()
860 		switch (group->default_domain->type) {  in iommu_group_show_type()
878 	mutex_unlock(&group->mutex);  in iommu_group_show_type()
893 	struct iommu_group *group = to_iommu_group(kobj);  in iommu_group_release()  local
895 	pr_debug("Releasing group %d\n", group->id);  in iommu_group_release()
897 	if (group->iommu_data_release)  in iommu_group_release()
898 		group->iommu_data_release(group->iommu_data);  in iommu_group_release()
900 	ida_free(&iommu_group_ida, group->id);  in iommu_group_release()
903 	WARN_ON(group->default_domain);  in iommu_group_release()
904 	WARN_ON(group->blocking_domain);  in iommu_group_release()
906 	kfree(group->name);  in iommu_group_release()
907 	kfree(group);  in iommu_group_release()
928 	struct iommu_group *group;  in iommu_group_alloc()  local
931 	group = kzalloc(sizeof(*group), GFP_KERNEL);  in iommu_group_alloc()
932 	if (!group)  in iommu_group_alloc()
935 	group->kobj.kset = iommu_group_kset;  in iommu_group_alloc()
936 	mutex_init(&group->mutex);  in iommu_group_alloc()
937 	INIT_LIST_HEAD(&group->devices);  in iommu_group_alloc()
938 	INIT_LIST_HEAD(&group->entry);  in iommu_group_alloc()
939 	xa_init(&group->pasid_array);  in iommu_group_alloc()
943 		kfree(group);  in iommu_group_alloc()
946 	group->id = ret;  in iommu_group_alloc()
948 	ret = kobject_init_and_add(&group->kobj, &iommu_group_ktype,  in iommu_group_alloc()
949 				   NULL, "%d", group->id);  in iommu_group_alloc()
951 		kobject_put(&group->kobj);  in iommu_group_alloc()
955 	group->devices_kobj = kobject_create_and_add("devices", &group->kobj);  in iommu_group_alloc()
956 	if (!group->devices_kobj) {  in iommu_group_alloc()
957 		kobject_put(&group->kobj); /* triggers .release & free */  in iommu_group_alloc()
966 	kobject_put(&group->kobj);  in iommu_group_alloc()
968 	ret = iommu_group_create_file(group,  in iommu_group_alloc()
971 		kobject_put(group->devices_kobj);  in iommu_group_alloc()
975 	ret = iommu_group_create_file(group, &iommu_group_attr_type);  in iommu_group_alloc()
977 		kobject_put(group->devices_kobj);  in iommu_group_alloc()
981 	pr_debug("Allocated group %d\n", group->id);  in iommu_group_alloc()
983 	return group;  in iommu_group_alloc()
995 void *iommu_group_get_iommudata(struct iommu_group *group)  in iommu_group_get_iommudata()  argument
997 	return group->iommu_data;  in iommu_group_get_iommudata()
1011 void iommu_group_set_iommudata(struct iommu_group *group, void *iommu_data,  in iommu_group_set_iommudata()  argument
1014 	group->iommu_data = iommu_data;  in iommu_group_set_iommudata()
1015 	group->iommu_data_release = release;  in iommu_group_set_iommudata()
1027 int iommu_group_set_name(struct iommu_group *group, const char *name)  in iommu_group_set_name()  argument
1031 	if (group->name) {  in iommu_group_set_name()
1032 		iommu_group_remove_file(group, &iommu_group_attr_name);  in iommu_group_set_name()
1033 		kfree(group->name);  in iommu_group_set_name()
1034 		group->name = NULL;  in iommu_group_set_name()
1039 	group->name = kstrdup(name, GFP_KERNEL);  in iommu_group_set_name()
1040 	if (!group->name)  in iommu_group_set_name()
1043 	ret = iommu_group_create_file(group, &iommu_group_attr_name);  in iommu_group_set_name()
1045 		kfree(group->name);  in iommu_group_set_name()
1046 		group->name = NULL;  in iommu_group_set_name()
1121 static struct group_device *iommu_group_alloc_device(struct iommu_group *group,  in iommu_group_alloc_device()  argument
1133 	ret = sysfs_create_link(&dev->kobj, &group->kobj, "iommu_group");  in iommu_group_alloc_device()
1144 	ret = sysfs_create_link_nowarn(group->devices_kobj,  in iommu_group_alloc_device()
1160 	trace_add_device_to_group(group->id, dev);  in iommu_group_alloc_device()
1162 	dev_info(dev, "Adding to iommu group %d\n", group->id);  in iommu_group_alloc_device()
1172 	dev_err(dev, "Failed to add to iommu group %d: %d\n", group->id, ret);  in iommu_group_alloc_device()
1184 int iommu_group_add_device(struct iommu_group *group, struct device *dev)  in iommu_group_add_device()  argument
1188 	gdev = iommu_group_alloc_device(group, dev);  in iommu_group_add_device()
1192 	iommu_group_ref_get(group);  in iommu_group_add_device()
1193 	dev->iommu_group = group;  in iommu_group_add_device()
1195 	mutex_lock(&group->mutex);  in iommu_group_add_device()
1196 	list_add_tail(&gdev->list, &group->devices);  in iommu_group_add_device()
1197 	mutex_unlock(&group->mutex);  in iommu_group_add_device()
1211 	struct iommu_group *group = dev->iommu_group;  in iommu_group_remove_device()  local
1213 	if (!group)  in iommu_group_remove_device()
1216 	dev_info(dev, "Removing from iommu group %d\n", group->id);  in iommu_group_remove_device()
1233 int iommu_group_for_each_dev(struct iommu_group *group, void *data,  in iommu_group_for_each_dev()  argument
1239 	mutex_lock(&group->mutex);  in iommu_group_for_each_dev()
1240 	for_each_group_device(group, device) {  in iommu_group_for_each_dev()
1245 	mutex_unlock(&group->mutex);  in iommu_group_for_each_dev()
1261 	struct iommu_group *group = dev->iommu_group;  in iommu_group_get()  local
1263 	if (group)  in iommu_group_get()
1264 		kobject_get(group->devices_kobj);  in iommu_group_get()
1266 	return group;  in iommu_group_get()
1277 struct iommu_group *iommu_group_ref_get(struct iommu_group *group)  in iommu_group_ref_get()  argument
1279 	kobject_get(group->devices_kobj);  in iommu_group_ref_get()
1280 	return group;  in iommu_group_ref_get()
1291 void iommu_group_put(struct iommu_group *group)  in iommu_group_put()  argument
1293 	if (group)  in iommu_group_put()
1294 		kobject_put(group->devices_kobj);  in iommu_group_put()
1515 int iommu_group_id(struct iommu_group *group)  in iommu_group_id()  argument
1517 	return group->id;  in iommu_group_id()
1544 	struct iommu_group *group;  in get_pci_function_alias_group()  local
1555 		group = get_pci_alias_group(tmp, devfns);  in get_pci_function_alias_group()
1556 		if (group) {  in get_pci_function_alias_group()
1558 			return group;  in get_pci_function_alias_group()
1578 	struct iommu_group *group;  in get_pci_alias_group()  local
1583 	group = iommu_group_get(&pdev->dev);  in get_pci_alias_group()
1584 	if (group)  in get_pci_alias_group()
1585 		return group;  in get_pci_alias_group()
1593 			group = get_pci_alias_group(tmp, devfns);  in get_pci_alias_group()
1594 			if (group) {  in get_pci_alias_group()
1596 				return group;  in get_pci_alias_group()
1599 			group = get_pci_function_alias_group(tmp, devfns);  in get_pci_alias_group()
1600 			if (group) {  in get_pci_alias_group()
1602 				return group;  in get_pci_alias_group()
1612 	struct iommu_group *group;  member
1624 	data->group = iommu_group_get(&pdev->dev);  in get_pci_alias_or_group()
1626 	return data->group != NULL;  in get_pci_alias_or_group()
1648 	struct iommu_group *group = NULL;  in pci_device_group()  local
1661 		return data.group;  in pci_device_group()
1680 		group = iommu_group_get(&pdev->dev);  in pci_device_group()
1681 		if (group)  in pci_device_group()
1682 			return group;  in pci_device_group()
1689 	group = get_pci_alias_group(pdev, (unsigned long *)devfns);  in pci_device_group()
1690 	if (group)  in pci_device_group()
1691 		return group;  in pci_device_group()
1698 	group = get_pci_function_alias_group(pdev, (unsigned long *)devfns);  in pci_device_group()
1699 	if (group)  in pci_device_group()
1700 		return group;  in pci_device_group()
1711 	struct iommu_group *group;  in fsl_mc_device_group()  local
1713 	group = iommu_group_get(cont_dev);  in fsl_mc_device_group()
1714 	if (!group)  in fsl_mc_device_group()
1715 		group = iommu_group_alloc();  in fsl_mc_device_group()
1716 	return group;  in fsl_mc_device_group()
1735 				   struct iommu_group *group, int req_type)  in __iommu_group_alloc_default_domain()  argument
1737 	if (group->default_domain && group->default_domain->type == req_type)  in __iommu_group_alloc_default_domain()
1738 		return group->default_domain;  in __iommu_group_alloc_default_domain()
1747 iommu_group_alloc_default_domain(struct iommu_group *group, int req_type)  in iommu_group_alloc_default_domain()  argument
1750 		list_first_entry(&group->devices, struct group_device, list)  in iommu_group_alloc_default_domain()
1754 	lockdep_assert_held(&group->mutex);  in iommu_group_alloc_default_domain()
1757 		return __iommu_group_alloc_default_domain(bus, group, req_type);  in iommu_group_alloc_default_domain()
1760 	dom = __iommu_group_alloc_default_domain(bus, group, iommu_def_domain_type);  in iommu_group_alloc_default_domain()
1767 	dom = __iommu_group_alloc_default_domain(bus, group, IOMMU_DOMAIN_DMA);  in iommu_group_alloc_default_domain()
1772 		iommu_def_domain_type, group->name);  in iommu_group_alloc_default_domain()
1776 struct iommu_domain *iommu_group_default_domain(struct iommu_group *group)  in iommu_group_default_domain()  argument
1778 	return group->default_domain;  in iommu_group_default_domain()
1812 static int iommu_get_default_domain_type(struct iommu_group *group,  in iommu_get_default_domain_type()  argument
1819 	lockdep_assert_held(&group->mutex);  in iommu_get_default_domain_type()
1821 	for_each_group_device(group, gdev) {  in iommu_get_default_domain_type()
1857 	struct iommu_group *group, *next;  in bus_iommu_probe()  local
1865 	list_for_each_entry_safe(group, next, &group_list, entry) {  in bus_iommu_probe()
1868 		mutex_lock(&group->mutex);  in bus_iommu_probe()
1871 		list_del_init(&group->entry);  in bus_iommu_probe()
1878 		ret = iommu_setup_default_domain(group, 0);  in bus_iommu_probe()
1880 			mutex_unlock(&group->mutex);  in bus_iommu_probe()
1883 		mutex_unlock(&group->mutex);  in bus_iommu_probe()
1891 		for_each_group_device(group, gdev)  in bus_iommu_probe()
1937 bool iommu_group_has_isolated_msi(struct iommu_group *group)  in iommu_group_has_isolated_msi()  argument
1942 	mutex_lock(&group->mutex);  in iommu_group_has_isolated_msi()
1943 	for_each_group_device(group, group_dev)  in iommu_group_has_isolated_msi()
1945 	mutex_unlock(&group->mutex);  in iommu_group_has_isolated_msi()
2023 static void __iommu_group_set_core_domain(struct iommu_group *group)  in __iommu_group_set_core_domain()  argument
2027 	if (group->owner)  in __iommu_group_set_core_domain()
2028 		new_domain = group->blocking_domain;  in __iommu_group_set_core_domain()
2030 		new_domain = group->default_domain;  in __iommu_group_set_core_domain()
2032 	__iommu_group_set_domain_nofail(group, new_domain);  in __iommu_group_set_core_domain()
2065 	struct iommu_group *group;  in iommu_attach_device()  local
2068 	group = iommu_group_get(dev);  in iommu_attach_device()
2069 	if (!group)  in iommu_attach_device()
2076 	mutex_lock(&group->mutex);  in iommu_attach_device()
2078 	if (list_count_nodes(&group->devices) != 1)  in iommu_attach_device()
2081 	ret = __iommu_attach_group(domain, group);  in iommu_attach_device()
2084 	mutex_unlock(&group->mutex);  in iommu_attach_device()
2085 	iommu_group_put(group);  in iommu_attach_device()
2101 	struct iommu_group *group;  in iommu_detach_device()  local
2103 	group = iommu_group_get(dev);  in iommu_detach_device()
2104 	if (!group)  in iommu_detach_device()
2107 	mutex_lock(&group->mutex);  in iommu_detach_device()
2108 	if (WARN_ON(domain != group->domain) ||  in iommu_detach_device()
2109 	    WARN_ON(list_count_nodes(&group->devices) != 1))  in iommu_detach_device()
2111 	__iommu_group_set_core_domain(group);  in iommu_detach_device()
2114 	mutex_unlock(&group->mutex);  in iommu_detach_device()
2115 	iommu_group_put(group);  in iommu_detach_device()
2122 	struct iommu_group *group;  in iommu_get_domain_for_dev()  local
2124 	group = iommu_group_get(dev);  in iommu_get_domain_for_dev()
2125 	if (!group)  in iommu_get_domain_for_dev()
2128 	domain = group->domain;  in iommu_get_domain_for_dev()
2130 	iommu_group_put(group);  in iommu_get_domain_for_dev()
2146 				struct iommu_group *group)  in __iommu_attach_group()  argument
2148 	if (group->domain && group->domain != group->default_domain &&  in __iommu_attach_group()
2149 	    group->domain != group->blocking_domain)  in __iommu_attach_group()
2152 	return __iommu_group_set_domain(group, domain);  in __iommu_attach_group()
2167 int iommu_attach_group(struct iommu_domain *domain, struct iommu_group *group)  in iommu_attach_group()  argument
2171 	mutex_lock(&group->mutex);  in iommu_attach_group()
2172 	ret = __iommu_attach_group(domain, group);  in iommu_attach_group()
2173 	mutex_unlock(&group->mutex);  in iommu_attach_group()
2190 int iommu_group_replace_domain(struct iommu_group *group,  in iommu_group_replace_domain()  argument
2198 	mutex_lock(&group->mutex);  in iommu_group_replace_domain()
2199 	ret = __iommu_group_set_domain(group, new_domain);  in iommu_group_replace_domain()
2200 	mutex_unlock(&group->mutex);  in iommu_group_replace_domain()
2205 static int __iommu_device_set_domain(struct iommu_group *group,  in __iommu_device_set_domain()  argument
2221 	     new_domain == group->blocking_domain)) {  in __iommu_device_set_domain()
2228 		if (new_domain == group->default_domain)  in __iommu_device_set_domain()
2241 		    group->blocking_domain &&  in __iommu_device_set_domain()
2242 		    group->blocking_domain != new_domain)  in __iommu_device_set_domain()
2243 			__iommu_attach_device(group->blocking_domain, dev);  in __iommu_device_set_domain()
2264 static int __iommu_group_set_domain_internal(struct iommu_group *group,  in __iommu_group_set_domain_internal()  argument
2273 	lockdep_assert_held(&group->mutex);  in __iommu_group_set_domain_internal()
2275 	if (group->domain == new_domain)  in __iommu_group_set_domain_internal()
2284 		for_each_group_device(group, gdev) {  in __iommu_group_set_domain_internal()
2290 		group->domain = NULL;  in __iommu_group_set_domain_internal()
2301 	for_each_group_device(group, gdev) {  in __iommu_group_set_domain_internal()
2302 		ret = __iommu_device_set_domain(group, gdev->dev, new_domain,  in __iommu_group_set_domain_internal()
2318 	group->domain = new_domain;  in __iommu_group_set_domain_internal()
2327 	for_each_group_device(group, gdev) {  in __iommu_group_set_domain_internal()
2335 		if (group->domain)  in __iommu_group_set_domain_internal()
2337 				group, gdev->dev, group->domain,  in __iommu_group_set_domain_internal()
2347 void iommu_detach_group(struct iommu_domain *domain, struct iommu_group *group)  in iommu_detach_group()  argument
2349 	mutex_lock(&group->mutex);  in iommu_detach_group()
2350 	__iommu_group_set_core_domain(group);  in iommu_detach_group()
2351 	mutex_unlock(&group->mutex);  in iommu_detach_group()
2944 static int iommu_setup_default_domain(struct iommu_group *group,  in iommu_setup_default_domain()  argument
2947 	struct iommu_domain *old_dom = group->default_domain;  in iommu_setup_default_domain()
2954 	lockdep_assert_held(&group->mutex);  in iommu_setup_default_domain()
2956 	req_type = iommu_get_default_domain_type(group, target_type);  in iommu_setup_default_domain()
2967 	dom = iommu_group_alloc_default_domain(group, req_type);  in iommu_setup_default_domain()
2970 		if (group->default_domain)  in iommu_setup_default_domain()
2972 		group->default_domain = NULL;  in iommu_setup_default_domain()
2976 	if (group->default_domain == dom)  in iommu_setup_default_domain()
2985 	for_each_group_device(group, gdev) {  in iommu_setup_default_domain()
2995 	group->default_domain = dom;  in iommu_setup_default_domain()
2996 	if (!group->domain) {  in iommu_setup_default_domain()
3004 			group, dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);  in iommu_setup_default_domain()
3008 		ret = __iommu_group_set_domain(group, dom);  in iommu_setup_default_domain()
3020 		for_each_group_device(group, gdev) {  in iommu_setup_default_domain()
3035 			group, old_dom, IOMMU_SET_DOMAIN_MUST_SUCCEED);  in iommu_setup_default_domain()
3039 		group->default_domain = old_dom;  in iommu_setup_default_domain()
3053 static ssize_t iommu_group_store_type(struct iommu_group *group,  in iommu_group_store_type()  argument
3062 	if (WARN_ON(!group) || !group->default_domain)  in iommu_group_store_type()
3076 	mutex_lock(&group->mutex);  in iommu_group_store_type()
3079 	    group->default_domain->type == IOMMU_DOMAIN_DMA) {  in iommu_group_store_type()
3080 		ret = iommu_dma_init_fq(group->default_domain);  in iommu_group_store_type()
3084 		group->default_domain->type = IOMMU_DOMAIN_DMA_FQ;  in iommu_group_store_type()
3090 	if (list_empty(&group->devices) || group->owner_cnt) {  in iommu_group_store_type()
3095 	ret = iommu_setup_default_domain(group, req_type);  in iommu_group_store_type()
3105 	mutex_unlock(&group->mutex);  in iommu_group_store_type()
3108 	for_each_group_device(group, gdev)  in iommu_group_store_type()
3113 	mutex_unlock(&group->mutex);  in iommu_group_store_type()
3117 static bool iommu_is_default_domain(struct iommu_group *group)  in iommu_is_default_domain()  argument
3119 	if (group->domain == group->default_domain)  in iommu_is_default_domain()
3128 	if (group->default_domain &&  in iommu_is_default_domain()
3129 	    group->default_domain->type == IOMMU_DOMAIN_IDENTITY &&  in iommu_is_default_domain()
3130 	    group->domain && group->domain->type == IOMMU_DOMAIN_IDENTITY)  in iommu_is_default_domain()
3145 	struct iommu_group *group = iommu_group_get(dev);  in iommu_device_use_default_domain()  local
3148 	if (!group)  in iommu_device_use_default_domain()
3151 	mutex_lock(&group->mutex);  in iommu_device_use_default_domain()
3152 	if (group->owner_cnt) {  in iommu_device_use_default_domain()
3153 		if (group->owner || !iommu_is_default_domain(group) ||  in iommu_device_use_default_domain()
3154 		    !xa_empty(&group->pasid_array)) {  in iommu_device_use_default_domain()
3160 	group->owner_cnt++;  in iommu_device_use_default_domain()
3163 	mutex_unlock(&group->mutex);  in iommu_device_use_default_domain()
3164 	iommu_group_put(group);  in iommu_device_use_default_domain()
3179 	struct iommu_group *group = iommu_group_get(dev);  in iommu_device_unuse_default_domain()  local
3181 	if (!group)  in iommu_device_unuse_default_domain()
3184 	mutex_lock(&group->mutex);  in iommu_device_unuse_default_domain()
3185 	if (!WARN_ON(!group->owner_cnt || !xa_empty(&group->pasid_array)))  in iommu_device_unuse_default_domain()
3186 		group->owner_cnt--;  in iommu_device_unuse_default_domain()
3188 	mutex_unlock(&group->mutex);  in iommu_device_unuse_default_domain()
3189 	iommu_group_put(group);  in iommu_device_unuse_default_domain()
3192 static int __iommu_group_alloc_blocking_domain(struct iommu_group *group)  in __iommu_group_alloc_blocking_domain()  argument
3195 		list_first_entry(&group->devices, struct group_device, list);  in __iommu_group_alloc_blocking_domain()
3197 	if (group->blocking_domain)  in __iommu_group_alloc_blocking_domain()
3200 	group->blocking_domain =  in __iommu_group_alloc_blocking_domain()
3202 	if (!group->blocking_domain) {  in __iommu_group_alloc_blocking_domain()
3207 		group->blocking_domain = __iommu_domain_alloc(  in __iommu_group_alloc_blocking_domain()
3209 		if (!group->blocking_domain)  in __iommu_group_alloc_blocking_domain()
3215 static int __iommu_take_dma_ownership(struct iommu_group *group, void *owner)  in __iommu_take_dma_ownership()  argument
3219 	if ((group->domain && group->domain != group->default_domain) ||  in __iommu_take_dma_ownership()
3220 	    !xa_empty(&group->pasid_array))  in __iommu_take_dma_ownership()
3223 	ret = __iommu_group_alloc_blocking_domain(group);  in __iommu_take_dma_ownership()
3226 	ret = __iommu_group_set_domain(group, group->blocking_domain);  in __iommu_take_dma_ownership()
3230 	group->owner = owner;  in __iommu_take_dma_ownership()
3231 	group->owner_cnt++;  in __iommu_take_dma_ownership()
3244 int iommu_group_claim_dma_owner(struct iommu_group *group, void *owner)  in iommu_group_claim_dma_owner()  argument
3251 	mutex_lock(&group->mutex);  in iommu_group_claim_dma_owner()
3252 	if (group->owner_cnt) {  in iommu_group_claim_dma_owner()
3257 	ret = __iommu_take_dma_ownership(group, owner);  in iommu_group_claim_dma_owner()
3259 	mutex_unlock(&group->mutex);  in iommu_group_claim_dma_owner()
3276 	struct iommu_group *group;  in iommu_device_claim_dma_owner()  local
3282 	group = iommu_group_get(dev);  in iommu_device_claim_dma_owner()
3283 	if (!group)  in iommu_device_claim_dma_owner()
3286 	mutex_lock(&group->mutex);  in iommu_device_claim_dma_owner()
3287 	if (group->owner_cnt) {  in iommu_device_claim_dma_owner()
3288 		if (group->owner != owner) {  in iommu_device_claim_dma_owner()
3292 		group->owner_cnt++;  in iommu_device_claim_dma_owner()
3296 	ret = __iommu_take_dma_ownership(group, owner);  in iommu_device_claim_dma_owner()
3298 	mutex_unlock(&group->mutex);  in iommu_device_claim_dma_owner()
3299 	iommu_group_put(group);  in iommu_device_claim_dma_owner()
3305 static void __iommu_release_dma_ownership(struct iommu_group *group)  in __iommu_release_dma_ownership()  argument
3307 	if (WARN_ON(!group->owner_cnt || !group->owner ||  in __iommu_release_dma_ownership()
3308 		    !xa_empty(&group->pasid_array)))  in __iommu_release_dma_ownership()
3311 	group->owner_cnt = 0;  in __iommu_release_dma_ownership()
3312 	group->owner = NULL;  in __iommu_release_dma_ownership()
3313 	__iommu_group_set_domain_nofail(group, group->default_domain);  in __iommu_release_dma_ownership()
3322 void iommu_group_release_dma_owner(struct iommu_group *group)  in iommu_group_release_dma_owner()  argument
3324 	mutex_lock(&group->mutex);  in iommu_group_release_dma_owner()
3325 	__iommu_release_dma_ownership(group);  in iommu_group_release_dma_owner()
3326 	mutex_unlock(&group->mutex);  in iommu_group_release_dma_owner()
3338 	struct iommu_group *group = iommu_group_get(dev);  in iommu_device_release_dma_owner()  local
3340 	mutex_lock(&group->mutex);  in iommu_device_release_dma_owner()
3341 	if (group->owner_cnt > 1)  in iommu_device_release_dma_owner()
3342 		group->owner_cnt--;  in iommu_device_release_dma_owner()
3344 		__iommu_release_dma_ownership(group);  in iommu_device_release_dma_owner()
3345 	mutex_unlock(&group->mutex);  in iommu_device_release_dma_owner()
3346 	iommu_group_put(group);  in iommu_device_release_dma_owner()
3357 bool iommu_group_dma_owner_claimed(struct iommu_group *group)  in iommu_group_dma_owner_claimed()  argument
3361 	mutex_lock(&group->mutex);  in iommu_group_dma_owner_claimed()
3362 	user = group->owner_cnt;  in iommu_group_dma_owner_claimed()
3363 	mutex_unlock(&group->mutex);  in iommu_group_dma_owner_claimed()
3370 				   struct iommu_group *group, ioasid_t pasid)  in __iommu_set_group_pasid()  argument
3375 	for_each_group_device(group, device) {  in __iommu_set_group_pasid()
3384 static void __iommu_remove_group_pasid(struct iommu_group *group,  in __iommu_remove_group_pasid()  argument
3390 	for_each_group_device(group, device) {  in __iommu_remove_group_pasid()
3407 	struct iommu_group *group;  in iommu_attach_device_pasid()  local
3414 	group = iommu_group_get(dev);  in iommu_attach_device_pasid()
3415 	if (!group)  in iommu_attach_device_pasid()
3418 	mutex_lock(&group->mutex);  in iommu_attach_device_pasid()
3419 	curr = xa_cmpxchg(&group->pasid_array, pasid, NULL, domain, GFP_KERNEL);  in iommu_attach_device_pasid()
3425 	ret = __iommu_set_group_pasid(domain, group, pasid);  in iommu_attach_device_pasid()
3427 		__iommu_remove_group_pasid(group, pasid);  in iommu_attach_device_pasid()
3428 		xa_erase(&group->pasid_array, pasid);  in iommu_attach_device_pasid()
3431 	mutex_unlock(&group->mutex);  in iommu_attach_device_pasid()
3432 	iommu_group_put(group);  in iommu_attach_device_pasid()
3450 	struct iommu_group *group = iommu_group_get(dev);  in iommu_detach_device_pasid()  local
3452 	mutex_lock(&group->mutex);  in iommu_detach_device_pasid()
3453 	__iommu_remove_group_pasid(group, pasid);  in iommu_detach_device_pasid()
3454 	WARN_ON(xa_erase(&group->pasid_array, pasid) != domain);  in iommu_detach_device_pasid()
3455 	mutex_unlock(&group->mutex);  in iommu_detach_device_pasid()
3457 	iommu_group_put(group);  in iommu_detach_device_pasid()
3480 	struct iommu_group *group;  in iommu_get_domain_for_dev_pasid()  local
3482 	group = iommu_group_get(dev);  in iommu_get_domain_for_dev_pasid()
3483 	if (!group)  in iommu_get_domain_for_dev_pasid()
3486 	xa_lock(&group->pasid_array);  in iommu_get_domain_for_dev_pasid()
3487 	domain = xa_load(&group->pasid_array, pasid);  in iommu_get_domain_for_dev_pasid()
3490 	xa_unlock(&group->pasid_array);  in iommu_get_domain_for_dev_pasid()
3491 	iommu_group_put(group);  in iommu_get_domain_for_dev_pasid()