31 					     struct dma_coherent_mem * mem)  in dma_get_device_base()  argument
33 	if (mem->use_dev_dma_pfn_offset)  in dma_get_device_base()
34 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));  in dma_get_device_base()
35 	return mem->device_base;  in dma_get_device_base()
77 static void _dma_release_coherent_memory(struct dma_coherent_mem *mem)  in _dma_release_coherent_memory()  argument
79 	if (!mem)  in _dma_release_coherent_memory()
82 	memunmap(mem->virt_base);  in _dma_release_coherent_memory()
83 	bitmap_free(mem->bitmap);  in _dma_release_coherent_memory()
84 	kfree(mem);  in _dma_release_coherent_memory()
88 				      struct dma_coherent_mem *mem)  in dma_assign_coherent_memory()  argument
96 	dev->dma_mem = mem;  in dma_assign_coherent_memory()
120 	struct dma_coherent_mem *mem;  in dma_declare_coherent_memory()  local
123 	mem = dma_init_coherent_memory(phys_addr, device_addr, size, false);  in dma_declare_coherent_memory()
124 	if (IS_ERR(mem))  in dma_declare_coherent_memory()
125 		return PTR_ERR(mem);  in dma_declare_coherent_memory()
127 	ret = dma_assign_coherent_memory(dev, mem);  in dma_declare_coherent_memory()
129 		_dma_release_coherent_memory(mem);  in dma_declare_coherent_memory()
140 				       struct dma_coherent_mem *mem,  in __dma_alloc_from_coherent()  argument
148 	spin_lock_irqsave(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
150 	if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))  in __dma_alloc_from_coherent()
153 	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);  in __dma_alloc_from_coherent()
160 	*dma_handle = dma_get_device_base(dev, mem) +  in __dma_alloc_from_coherent()
162 	ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);  in __dma_alloc_from_coherent()
163 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
167 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
188 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_alloc_from_dev_coherent()  local
190 	if (!mem)  in dma_alloc_from_dev_coherent()
193 	*ret = __dma_alloc_from_coherent(dev, mem, size, dma_handle);  in dma_alloc_from_dev_coherent()
197 static int __dma_release_from_coherent(struct dma_coherent_mem *mem,  in __dma_release_from_coherent()  argument
200 	if (mem && vaddr >= mem->virt_base && vaddr <  in __dma_release_from_coherent()
201 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_release_from_coherent()
202 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_release_from_coherent()
205 		spin_lock_irqsave(&mem->spinlock, flags);  in __dma_release_from_coherent()
206 		bitmap_release_region(mem->bitmap, page, order);  in __dma_release_from_coherent()
207 		spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_release_from_coherent()
227 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_release_from_dev_coherent()  local
229 	return __dma_release_from_coherent(mem, order, vaddr);  in dma_release_from_dev_coherent()
232 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,  in __dma_mmap_from_coherent()  argument
235 	if (mem && vaddr >= mem->virt_base && vaddr + size <=  in __dma_mmap_from_coherent()
236 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_mmap_from_coherent()
238 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_mmap_from_coherent()
244 			unsigned long pfn = mem->pfn_base + start + off;  in __dma_mmap_from_coherent()
272 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_mmap_from_dev_coherent()  local
274 	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);  in dma_mmap_from_dev_coherent()
311 	struct dma_coherent_mem *mem;  in dma_init_global_coherent()  local
313 	mem = dma_init_coherent_memory(phys_addr, phys_addr, size, true);  in dma_init_global_coherent()
314 	if (IS_ERR(mem))  in dma_init_global_coherent()
315 		return PTR_ERR(mem);  in dma_init_global_coherent()
316 	dma_coherent_default_memory = mem;  in dma_init_global_coherent()
337 		struct dma_coherent_mem *mem;  in rmem_dma_device_init()  local
339 		mem = dma_init_coherent_memory(rmem->base, rmem->base,  in rmem_dma_device_init()
341 		if (IS_ERR(mem))  in rmem_dma_device_init()
342 			return PTR_ERR(mem);  in rmem_dma_device_init()
343 		rmem->priv = mem;  in rmem_dma_device_init()