296 static struct dmar_domain *si_domain;
340 static void domain_exit(struct dmar_domain *domain);
341 static void domain_remove_dev_info(struct dmar_domain *domain);
346 static int domain_detach_iommu(struct dmar_domain *domain,
432 static struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)  in to_dmar_domain()
434 	return container_of(dom, struct dmar_domain, domain);  in to_dmar_domain()
484 static struct dmar_domain* get_iommu_domain(struct intel_iommu *iommu, u16 did)  in get_iommu_domain()
486 	struct dmar_domain **domains;  in get_iommu_domain()
497 			     struct dmar_domain *domain)  in set_iommu_domain()
499 	struct dmar_domain **domains;  in set_iommu_domain()
503 		size_t size = 256 * sizeof(struct dmar_domain *);  in set_iommu_domain()
550 static inline int domain_type_is_si(struct dmar_domain *domain)  in domain_type_is_si()
555 static inline int domain_pfn_supported(struct dmar_domain *domain,  in domain_pfn_supported()
597 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)  in domain_get_iommu()
614 static void domain_update_iommu_coherency(struct dmar_domain *domain)  in domain_update_iommu_coherency()
689 static void domain_update_iommu_cap(struct dmar_domain *domain)  in domain_update_iommu_cap()
831 static void domain_flush_cache(struct dmar_domain *domain,  in domain_flush_cache()
881 static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,  in pfn_to_dma_pte()
936 static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,  in dma_pfn_level_pte()
968 static void dma_pte_clear_range(struct dmar_domain *domain,  in dma_pte_clear_range()
999 static void dma_pte_free_level(struct dmar_domain *domain, int level,  in dma_pte_free_level()
1042 static void dma_pte_free_pagetable(struct dmar_domain *domain,  in dma_pte_free_pagetable()
1070 static struct page *dma_pte_list_pagetables(struct dmar_domain *domain,  in dma_pte_list_pagetables()
1094 static struct page *dma_pte_clear_level(struct dmar_domain *domain, int level,  in dma_pte_clear_level()
1146 static struct page *domain_unmap(struct dmar_domain *domain,  in domain_unmap()
1343 iommu_support_dev_iotlb (struct dmar_domain *domain, struct intel_iommu *iommu,  in iommu_support_dev_iotlb()
1364 static void domain_update_iotlb(struct dmar_domain *domain)  in domain_update_iotlb()
1463 static void iommu_flush_dev_iotlb(struct dmar_domain *domain,  in iommu_flush_dev_iotlb()
1487 				  struct dmar_domain *domain,  in iommu_flush_iotlb_psi()
1522 					struct dmar_domain *domain,  in __mapping_notify_one()
1534 	struct dmar_domain *domain;  in iommu_flush_iova()
1537 	domain = container_of(iovad, struct dmar_domain, iovad);  in iommu_flush_iova()
1622 	size = (ALIGN(ndomains, 256) >> 8) * sizeof(struct dmar_domain **);  in iommu_init_domains()
1626 		size = 256 * sizeof(struct dmar_domain *);  in iommu_init_domains()
1712 static struct dmar_domain *alloc_domain(int flags)  in alloc_domain()
1714 	struct dmar_domain *domain;  in alloc_domain()
1730 static int domain_attach_iommu(struct dmar_domain *domain,  in domain_attach_iommu()
1764 static int domain_detach_iommu(struct dmar_domain *domain,  in domain_detach_iommu()
1828 static void domain_reserve_special_ranges(struct dmar_domain *domain)  in domain_reserve_special_ranges()
1847 static int domain_init(struct dmar_domain *domain, struct intel_iommu *iommu,  in domain_init()
1904 static void domain_exit(struct dmar_domain *domain)  in domain_exit()
1973 static int domain_context_mapping_one(struct dmar_domain *domain,  in domain_context_mapping_one()
2122 	struct dmar_domain *domain;
2138 domain_context_mapping(struct dmar_domain *domain, struct device *dev)  in domain_context_mapping()
2196 static inline int hardware_largepage_caps(struct dmar_domain *domain,  in hardware_largepage_caps()
2223 static int __domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in __domain_mapping()
2338 static int domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_mapping()
2358 static inline int domain_sg_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_sg_mapping()
2365 static inline int domain_pfn_mapping(struct dmar_domain *domain, unsigned long iov_pfn,  in domain_pfn_mapping()
2412 static void domain_remove_dev_info(struct dmar_domain *domain)  in domain_remove_dev_info()
2427 static struct dmar_domain *find_domain(struct device *dev)  in find_domain()
2461 static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu,  in dmar_insert_one_dev_info()
2464 						    struct dmar_domain *domain)  in dmar_insert_one_dev_info()
2466 	struct dmar_domain *found = NULL;  in dmar_insert_one_dev_info()
2586 static struct dmar_domain *find_or_alloc_domain(struct device *dev, int gaw)  in find_or_alloc_domain()
2589 	struct dmar_domain *domain = NULL;  in find_or_alloc_domain()
2632 static struct dmar_domain *set_domain_for_dev(struct device *dev,  in set_domain_for_dev()
2633 					      struct dmar_domain *domain)  in set_domain_for_dev()
2636 	struct dmar_domain *tmp;  in set_domain_for_dev()
2668 static int iommu_domain_identity_map(struct dmar_domain *domain,  in iommu_domain_identity_map()
2694 				       struct dmar_domain *domain,  in domain_prepare_identity_map()
2732 static int md_domain_init(struct dmar_domain *domain, int guest_width);
2803 static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)  in domain_add_dev_info()
2805 	struct dmar_domain *ndomain;  in domain_add_dev_info()
3380 				     struct dmar_domain *domain,  in intel_alloc_iova()
3411 static struct dmar_domain *get_private_domain_for_dev(struct device *dev)  in get_private_domain_for_dev()
3413 	struct dmar_domain *domain, *tmp;  in get_private_domain_for_dev()
3485 			struct dmar_domain *dmar_domain;  in iommu_need_mapping()  local
3489 				dmar_domain = to_dmar_domain(domain);  in iommu_need_mapping()
3490 				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;  in iommu_need_mapping()
3505 	struct dmar_domain *domain;  in __intel_map_single()
3584 	struct dmar_domain *domain;  in intel_unmap()
3722 	struct dmar_domain *domain;  in intel_map_sg()
3804 	struct dmar_domain *domain;  in bounce_sync_single()
3822 	struct dmar_domain *domain;  in bounce_map_single()
3908 	struct dmar_domain *domain;  in bounce_unmap_single()
4051 					 sizeof(struct dmar_domain),  in iommu_domain_cache_init()
4731 		struct dmar_domain *domain;  in free_all_cpu_cached_iovas()
5091 	struct dmar_domain *domain;  in __dmar_remove_one_dev_info()
5140 static int md_domain_init(struct dmar_domain *domain, int guest_width)  in md_domain_init()
5167 	struct dmar_domain *dmar_domain;  in intel_iommu_domain_alloc()  local
5174 		dmar_domain = alloc_domain(0);  in intel_iommu_domain_alloc()
5175 		if (!dmar_domain) {  in intel_iommu_domain_alloc()
5179 		if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {  in intel_iommu_domain_alloc()
5181 			domain_exit(dmar_domain);  in intel_iommu_domain_alloc()
5186 		    init_iova_flush_queue(&dmar_domain->iovad,  in intel_iommu_domain_alloc()
5192 		domain_update_iommu_cap(dmar_domain);  in intel_iommu_domain_alloc()
5194 		domain = &dmar_domain->domain;  in intel_iommu_domain_alloc()
5197 				__DOMAIN_MAX_ADDR(dmar_domain->gaw);  in intel_iommu_domain_alloc()
5229 static void auxiliary_link_device(struct dmar_domain *domain,  in auxiliary_link_device()
5242 static void auxiliary_unlink_device(struct dmar_domain *domain,  in auxiliary_unlink_device()
5258 static int aux_domain_add_dev(struct dmar_domain *domain,  in aux_domain_add_dev()
5317 static void aux_domain_remove_dev(struct dmar_domain *domain,  in aux_domain_remove_dev()
5344 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in prepare_domain_attach_device()  local
5358 	if (dmar_domain->max_addr > (1LL << addr_width)) {  in prepare_domain_attach_device()
5361 		        __func__, addr_width, dmar_domain->max_addr);  in prepare_domain_attach_device()
5364 	dmar_domain->gaw = addr_width;  in prepare_domain_attach_device()
5369 	while (iommu->agaw < dmar_domain->agaw) {  in prepare_domain_attach_device()
5372 		pte = dmar_domain->pgd;  in prepare_domain_attach_device()
5374 			dmar_domain->pgd = (struct dma_pte *)  in prepare_domain_attach_device()
5378 		dmar_domain->agaw--;  in prepare_domain_attach_device()
5400 		struct dmar_domain *old_domain;  in intel_iommu_attach_device()
5445 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_map()  local
5450 	if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN)  in intel_iommu_map()
5457 	if ((iommu_prot & IOMMU_CACHE) && dmar_domain->iommu_snooping)  in intel_iommu_map()
5461 	if (dmar_domain->max_addr < max_addr) {  in intel_iommu_map()
5465 		end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;  in intel_iommu_map()
5469 			       __func__, dmar_domain->gaw, max_addr);  in intel_iommu_map()
5472 		dmar_domain->max_addr = max_addr;  in intel_iommu_map()
5477 	ret = domain_pfn_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,  in intel_iommu_map()
5486 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_unmap()  local
5494 	BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));  in intel_iommu_unmap()
5495 	if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN)  in intel_iommu_unmap()
5504 	freelist = domain_unmap(dmar_domain, start_pfn, last_pfn);  in intel_iommu_unmap()
5508 	for_each_domain_iommu(iommu_id, dmar_domain)  in intel_iommu_unmap()
5509 		iommu_flush_iotlb_psi(g_iommus[iommu_id], dmar_domain,  in intel_iommu_unmap()
5514 	if (dmar_domain->max_addr == iova + size)  in intel_iommu_unmap()
5515 		dmar_domain->max_addr = iova;  in intel_iommu_unmap()
5523 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_iova_to_phys()  local
5528 	if (dmar_domain->flags & DOMAIN_FLAG_LOSE_CHILDREN)  in intel_iommu_iova_to_phys()
5531 	pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);  in intel_iommu_iova_to_phys()
5586 	struct dmar_domain *dmar_domain;  in intel_iommu_add_device()  local
5610 	dmar_domain = to_dmar_domain(domain);  in intel_iommu_add_device()
5616 				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;  in intel_iommu_add_device()
5627 				dmar_domain->flags |= DOMAIN_FLAG_LOSE_CHILDREN;  in intel_iommu_add_device()
5737 	struct dmar_domain *domain;  in intel_iommu_enable_pasid()
5788 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_apply_resv_region()  local
5794 	WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));  in intel_iommu_apply_resv_region()
5940 	struct dmar_domain *dmar_domain = to_dmar_domain(domain);  in intel_iommu_aux_get_pasid()  local
5942 	return dmar_domain->default_pasid > 0 ?  in intel_iommu_aux_get_pasid()
5943 			dmar_domain->default_pasid : -EINVAL;  in intel_iommu_aux_get_pasid()