52 static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)  in cookie_msi_granule()  argument
54 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE)  in cookie_msi_granule()
55 		return cookie->iovad.granule;  in cookie_msi_granule()
61 	struct iommu_dma_cookie *cookie;  in cookie_alloc()  local
63 	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);  in cookie_alloc()
64 	if (cookie) {  in cookie_alloc()
65 		spin_lock_init(&cookie->msi_lock);  in cookie_alloc()
66 		INIT_LIST_HEAD(&cookie->msi_page_list);  in cookie_alloc()
67 		cookie->type = type;  in cookie_alloc()
69 	return cookie;  in cookie_alloc()
106 	struct iommu_dma_cookie *cookie;  in iommu_get_msi_cookie()  local
114 	cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);  in iommu_get_msi_cookie()
115 	if (!cookie)  in iommu_get_msi_cookie()
118 	cookie->msi_iova = base;  in iommu_get_msi_cookie()
119 	domain->iova_cookie = cookie;  in iommu_get_msi_cookie()
133 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_put_dma_cookie()  local
136 	if (!cookie)  in iommu_put_dma_cookie()
139 	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)  in iommu_put_dma_cookie()
140 		put_iova_domain(&cookie->iovad);  in iommu_put_dma_cookie()
142 	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {  in iommu_put_dma_cookie()
146 	kfree(cookie);  in iommu_put_dma_cookie()
170 static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,  in cookie_init_hw_msi_region()  argument
173 	struct iova_domain *iovad = &cookie->iovad;  in cookie_init_hw_msi_region()
188 		list_add(&msi_page[i].list, &cookie->msi_page_list);  in cookie_init_hw_msi_region()
241 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iova_reserve_iommu_regions()  local
242 	struct iova_domain *iovad = &cookie->iovad;  in iova_reserve_iommu_regions()
266 			ret = cookie_init_hw_msi_region(cookie, region->start,  in iova_reserve_iommu_regions()
278 	struct iommu_dma_cookie *cookie;  in iommu_dma_flush_iotlb_all()  local
281 	cookie = container_of(iovad, struct iommu_dma_cookie, iovad);  in iommu_dma_flush_iotlb_all()
282 	domain = cookie->fq_domain;  in iommu_dma_flush_iotlb_all()
305 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_init_domain()  local
310 	if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)  in iommu_dma_init_domain()
313 	iovad = &cookie->iovad;  in iommu_dma_init_domain()
344 	if (!cookie->fq_domain && !iommu_domain_get_attr(domain,  in iommu_dma_init_domain()
346 		cookie->fq_domain = domain;  in iommu_dma_init_domain()
388 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_iova()  local
389 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_iova()
392 	if (cookie->type == IOMMU_DMA_MSI_COOKIE) {  in iommu_dma_alloc_iova()
393 		cookie->msi_iova += size;  in iommu_dma_alloc_iova()
394 		return cookie->msi_iova - size;  in iommu_dma_alloc_iova()
426 static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,  in iommu_dma_free_iova()  argument
429 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_free_iova()
432 	if (cookie->type == IOMMU_DMA_MSI_COOKIE)  in iommu_dma_free_iova()
433 		cookie->msi_iova -= size;  in iommu_dma_free_iova()
434 	else if (cookie->fq_domain)	/* non-strict mode */  in iommu_dma_free_iova()
446 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_unmap()  local
447 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_unmap()
459 	if (!cookie->fq_domain)  in __iommu_dma_unmap()
461 	iommu_dma_free_iova(cookie, dma_addr, size);  in __iommu_dma_unmap()
468 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in __iommu_dma_map()  local
469 	struct iova_domain *iovad = &cookie->iovad;  in __iommu_dma_map()
480 		iommu_dma_free_iova(cookie, iova, size);  in __iommu_dma_map()
569 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_alloc_remap()  local
570 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_alloc_remap()
632 	iommu_dma_free_iova(cookie, iova, size);  in iommu_dma_alloc_remap()
815 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_map_sg()  local
816 	struct iova_domain *iovad = &cookie->iovad;  in iommu_dma_map_sg()
880 	iommu_dma_free_iova(cookie, iova, iova_len);  in iommu_dma_map_sg()
1142 	struct iommu_dma_cookie *cookie = domain->iova_cookie;  in iommu_dma_get_msi_page()  local
1146 	size_t size = cookie_msi_granule(cookie);  in iommu_dma_get_msi_page()
1149 	list_for_each_entry(msi_page, &cookie->msi_page_list, list)  in iommu_dma_get_msi_page()
1167 	list_add(&msi_page->list, &cookie->msi_page_list);  in iommu_dma_get_msi_page()
1171 	iommu_dma_free_iova(cookie, iova, size);  in iommu_dma_get_msi_page()
1181 	struct iommu_dma_cookie *cookie;  in iommu_dma_prepare_msi()  local
1190 	cookie = domain->iova_cookie;  in iommu_dma_prepare_msi()
1197 	spin_lock_irqsave(&cookie->msi_lock, flags);  in iommu_dma_prepare_msi()
1199 	spin_unlock_irqrestore(&cookie->msi_lock, flags);  in iommu_dma_prepare_msi()