143 	struct iommu_dev_data *dev_data;  in alloc_dev_data()  local
145 	dev_data = kzalloc(sizeof(*dev_data), GFP_KERNEL);  in alloc_dev_data()
146 	if (!dev_data)  in alloc_dev_data()
149 	spin_lock_init(&dev_data->lock);  in alloc_dev_data()
150 	dev_data->devid = devid;  in alloc_dev_data()
151 	ratelimit_default_init(&dev_data->rs);  in alloc_dev_data()
153 	llist_add(&dev_data->dev_data_list, &dev_data_list);  in alloc_dev_data()
154 	return dev_data;  in alloc_dev_data()
159 	struct iommu_dev_data *dev_data;  in search_dev_data()  local
166 	llist_for_each_entry(dev_data, node, dev_data_list) {  in search_dev_data()
167 		if (dev_data->devid == devid)  in search_dev_data()
168 			return dev_data;  in search_dev_data()
230 	struct iommu_dev_data *dev_data;  in find_dev_data()  local
233 	dev_data = search_dev_data(devid);  in find_dev_data()
235 	if (dev_data == NULL) {  in find_dev_data()
236 		dev_data = alloc_dev_data(devid);  in find_dev_data()
237 		if (!dev_data)  in find_dev_data()
241 			dev_data->defer_attach = true;  in find_dev_data()
244 	return dev_data;  in find_dev_data()
319 	struct iommu_dev_data *dev_data;  in iommu_init_device()  local
329 	dev_data = find_dev_data(devid);  in iommu_init_device()
330 	if (!dev_data)  in iommu_init_device()
333 	dev_data->pdev = setup_aliases(dev);  in iommu_init_device()
345 		iommu = amd_iommu_rlookup_table[dev_data->devid];  in iommu_init_device()
346 		dev_data->iommu_v2 = iommu->is_iommu_v2;  in iommu_init_device()
349 	dev_iommu_priv_set(dev, dev_data);  in iommu_init_device()
370 	struct iommu_dev_data *dev_data;  in amd_iommu_uninit_device()  local
372 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_uninit_device()
373 	if (!dev_data)  in amd_iommu_uninit_device()
376 	if (dev_data->domain)  in amd_iommu_uninit_device()
413 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_hw_error()  local
426 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_hw_error()
428 	if (dev_data) {  in amd_iommu_report_rmp_hw_error()
429 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_hw_error()
445 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_rmp_fault()  local
459 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_rmp_fault()
461 	if (dev_data) {  in amd_iommu_report_rmp_fault()
462 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_rmp_fault()
479 	struct iommu_dev_data *dev_data = NULL;  in amd_iommu_report_page_fault()  local
485 		dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_report_page_fault()
487 	if (dev_data) {  in amd_iommu_report_page_fault()
488 		if (__ratelimit(&dev_data->rs)) {  in amd_iommu_report_page_fault()
1179 static int device_flush_iotlb(struct iommu_dev_data *dev_data,  in device_flush_iotlb()  argument
1186 	qdep     = dev_data->ats.qdep;  in device_flush_iotlb()
1187 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_iotlb()
1189 	build_inv_iotlb_pages(&cmd, dev_data->devid, qdep, address, size);  in device_flush_iotlb()
1204 static int device_flush_dte(struct iommu_dev_data *dev_data)  in device_flush_dte()  argument
1210 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in device_flush_dte()
1212 	if (dev_data->pdev)  in device_flush_dte()
1213 		ret = pci_for_each_dma_alias(dev_data->pdev,  in device_flush_dte()
1216 		ret = iommu_flush_dte(iommu, dev_data->devid);  in device_flush_dte()
1220 	alias = amd_iommu_alias_table[dev_data->devid];  in device_flush_dte()
1221 	if (alias != dev_data->devid) {  in device_flush_dte()
1227 	if (dev_data->ats.enabled)  in device_flush_dte()
1228 		ret = device_flush_iotlb(dev_data, 0, ~0UL);  in device_flush_dte()
1241 	struct iommu_dev_data *dev_data;  in __domain_flush_pages()  local
1258 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __domain_flush_pages()
1260 		if (!dev_data->ats.enabled)  in __domain_flush_pages()
1263 		ret |= device_flush_iotlb(dev_data, address, size);  in __domain_flush_pages()
1354 	struct iommu_dev_data *dev_data;  in domain_flush_devices()  local
1356 	list_for_each_entry(dev_data, &domain->dev_list, list)  in domain_flush_devices()
1357 		device_flush_dte(dev_data);  in domain_flush_devices()
1516 static void do_attach(struct iommu_dev_data *dev_data,  in do_attach()  argument
1522 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_attach()
1523 	ats   = dev_data->ats.enabled;  in do_attach()
1526 	dev_data->domain = domain;  in do_attach()
1527 	list_add(&dev_data->list, &domain->dev_list);  in do_attach()
1534 	set_dte_entry(dev_data->devid, domain,  in do_attach()
1535 		      ats, dev_data->iommu_v2);  in do_attach()
1536 	clone_aliases(dev_data->pdev);  in do_attach()
1538 	device_flush_dte(dev_data);  in do_attach()
1541 static void do_detach(struct iommu_dev_data *dev_data)  in do_detach()  argument
1543 	struct protection_domain *domain = dev_data->domain;  in do_detach()
1546 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in do_detach()
1549 	dev_data->domain = NULL;  in do_detach()
1550 	list_del(&dev_data->list);  in do_detach()
1551 	clear_dte_entry(dev_data->devid);  in do_detach()
1552 	clone_aliases(dev_data->pdev);  in do_detach()
1555 	device_flush_dte(dev_data);  in do_detach()
1615 	struct iommu_dev_data *dev_data;  in attach_device()  local
1622 	dev_data = dev_iommu_priv_get(dev);  in attach_device()
1624 	spin_lock(&dev_data->lock);  in attach_device()
1627 	if (dev_data->domain != NULL)  in attach_device()
1641 		if (dev_data->iommu_v2) {  in attach_device()
1645 			dev_data->ats.enabled = true;  in attach_device()
1646 			dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
1647 			dev_data->pri_tlp     = pci_prg_resp_pasid_required(pdev);  in attach_device()
1651 		dev_data->ats.enabled = true;  in attach_device()
1652 		dev_data->ats.qdep    = pci_ats_queue_depth(pdev);  in attach_device()
1658 	do_attach(dev_data, domain);  in attach_device()
1670 	spin_unlock(&dev_data->lock);  in attach_device()
1683 	struct iommu_dev_data *dev_data;  in detach_device()  local
1686 	dev_data = dev_iommu_priv_get(dev);  in detach_device()
1687 	domain   = dev_data->domain;  in detach_device()
1691 	spin_lock(&dev_data->lock);  in detach_device()
1699 	if (WARN_ON(!dev_data->domain))  in detach_device()
1702 	do_detach(dev_data);  in detach_device()
1707 	if (domain->flags & PD_IOMMUV2_MASK && dev_data->iommu_v2)  in detach_device()
1709 	else if (dev_data->ats.enabled)  in detach_device()
1712 	dev_data->ats.enabled = false;  in detach_device()
1715 	spin_unlock(&dev_data->lock);  in detach_device()
1788 	struct iommu_dev_data *dev_data;  in update_device_table()  local
1790 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in update_device_table()
1791 		set_dte_entry(dev_data->devid, domain,  in update_device_table()
1792 			      dev_data->ats.enabled, dev_data->iommu_v2);  in update_device_table()
1793 		clone_aliases(dev_data->pdev);  in update_device_table()
1985 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_detach_device()  local
1992 	if (dev_data->domain != NULL)  in amd_iommu_detach_device()
2002 		dev_data->use_vapic = 0;  in amd_iommu_detach_device()
2012 	struct iommu_dev_data *dev_data;  in amd_iommu_attach_device()  local
2019 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_attach_device()
2020 	dev_data->defer_attach = false;  in amd_iommu_attach_device()
2022 	iommu = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_attach_device()
2026 	if (dev_data->domain)  in amd_iommu_attach_device()
2034 			dev_data->use_vapic = 1;  in amd_iommu_attach_device()
2036 			dev_data->use_vapic = 0;  in amd_iommu_attach_device()
2199 	struct iommu_dev_data *dev_data = dev_iommu_priv_get(dev);  in amd_iommu_is_attach_deferred()  local
2201 	return dev_data->defer_attach;  in amd_iommu_is_attach_deferred()
2230 	struct iommu_dev_data *dev_data;  in amd_iommu_def_domain_type()  local
2232 	dev_data = dev_iommu_priv_get(dev);  in amd_iommu_def_domain_type()
2233 	if (!dev_data)  in amd_iommu_def_domain_type()
2241 	if (!mem_encrypt_active() && dev_data->iommu_v2)  in amd_iommu_def_domain_type()
2353 	struct iommu_dev_data *dev_data;  in __flush_pasid()  local
2379 	list_for_each_entry(dev_data, &domain->dev_list, list) {  in __flush_pasid()
2387 		if (!dev_data->ats.enabled)  in __flush_pasid()
2390 		qdep  = dev_data->ats.qdep;  in __flush_pasid()
2391 		iommu = amd_iommu_rlookup_table[dev_data->devid];  in __flush_pasid()
2393 		build_inv_iotlb_pasid(&cmd, dev_data->devid, pasid,  in __flush_pasid()
2549 	struct iommu_dev_data *dev_data;  in amd_iommu_complete_ppr()  local
2553 	dev_data = dev_iommu_priv_get(&pdev->dev);  in amd_iommu_complete_ppr()
2554 	iommu    = amd_iommu_rlookup_table[dev_data->devid];  in amd_iommu_complete_ppr()
2556 	build_complete_ppr(&cmd, dev_data->devid, pasid, status,  in amd_iommu_complete_ppr()
2557 			   tag, dev_data->pri_tlp);  in amd_iommu_complete_ppr()
3376 	struct iommu_dev_data *dev_data = search_dev_data(irte_info->devid);  in amd_ir_set_vcpu_affinity()  local
3382 	if (!dev_data || !dev_data->use_vapic)  in amd_ir_set_vcpu_affinity()