31 					     struct dma_coherent_mem * mem)  in dma_get_device_base()  argument
33 	if (mem->use_dev_dma_pfn_offset)  in dma_get_device_base()
34 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));  in dma_get_device_base()
35 	return mem->device_base;  in dma_get_device_base()
78 static void dma_release_coherent_memory(struct dma_coherent_mem *mem)  in dma_release_coherent_memory()  argument
80 	if (!mem)  in dma_release_coherent_memory()
83 	memunmap(mem->virt_base);  in dma_release_coherent_memory()
84 	kfree(mem->bitmap);  in dma_release_coherent_memory()
85 	kfree(mem);  in dma_release_coherent_memory()
89 				      struct dma_coherent_mem *mem)  in dma_assign_coherent_memory()  argument
97 	dev->dma_mem = mem;  in dma_assign_coherent_memory()
121 	struct dma_coherent_mem *mem;  in dma_declare_coherent_memory()  local
124 	mem = dma_init_coherent_memory(phys_addr, device_addr, size, false);  in dma_declare_coherent_memory()
125 	if (IS_ERR(mem))  in dma_declare_coherent_memory()
126 		return PTR_ERR(mem);  in dma_declare_coherent_memory()
128 	ret = dma_assign_coherent_memory(dev, mem);  in dma_declare_coherent_memory()
130 		dma_release_coherent_memory(mem);  in dma_declare_coherent_memory()
135 				       struct dma_coherent_mem *mem,  in __dma_alloc_from_coherent()  argument
143 	spin_lock_irqsave(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
145 	if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))  in __dma_alloc_from_coherent()
148 	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);  in __dma_alloc_from_coherent()
155 	*dma_handle = dma_get_device_base(dev, mem) +  in __dma_alloc_from_coherent()
157 	ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);  in __dma_alloc_from_coherent()
158 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
162 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
183 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_alloc_from_dev_coherent()  local
185 	if (!mem)  in dma_alloc_from_dev_coherent()
188 	*ret = __dma_alloc_from_coherent(dev, mem, size, dma_handle);  in dma_alloc_from_dev_coherent()
192 static int __dma_release_from_coherent(struct dma_coherent_mem *mem,  in __dma_release_from_coherent()  argument
195 	if (mem && vaddr >= mem->virt_base && vaddr <  in __dma_release_from_coherent()
196 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_release_from_coherent()
197 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_release_from_coherent()
200 		spin_lock_irqsave(&mem->spinlock, flags);  in __dma_release_from_coherent()
201 		bitmap_release_region(mem->bitmap, page, order);  in __dma_release_from_coherent()
202 		spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_release_from_coherent()
222 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_release_from_dev_coherent()  local
224 	return __dma_release_from_coherent(mem, order, vaddr);  in dma_release_from_dev_coherent()
227 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,  in __dma_mmap_from_coherent()  argument
230 	if (mem && vaddr >= mem->virt_base && vaddr + size <=  in __dma_mmap_from_coherent()
231 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_mmap_from_coherent()
233 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_mmap_from_coherent()
239 			unsigned long pfn = mem->pfn_base + start + off;  in __dma_mmap_from_coherent()
267 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_mmap_from_dev_coherent()  local
269 	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);  in dma_mmap_from_dev_coherent()
306 	struct dma_coherent_mem *mem;  in dma_init_global_coherent()  local
308 	mem = dma_init_coherent_memory(phys_addr, phys_addr, size, true);  in dma_init_global_coherent()
309 	if (IS_ERR(mem))  in dma_init_global_coherent()
310 		return PTR_ERR(mem);  in dma_init_global_coherent()
311 	dma_coherent_default_memory = mem;  in dma_init_global_coherent()
332 		struct dma_coherent_mem *mem;  in rmem_dma_device_init()  local
334 		mem = dma_init_coherent_memory(rmem->base, rmem->base,  in rmem_dma_device_init()
336 		if (IS_ERR(mem))  in rmem_dma_device_init()
337 			return PTR_ERR(mem);  in rmem_dma_device_init()
338 		rmem->priv = mem;  in rmem_dma_device_init()