1 // SPDX-License-Identifier: GPL-2.0
3  * Coherent per-device memory handling.
10 #include <linux/dma-direct.h>
11 #include <linux/dma-map-ops.h>
27 	if (dev && dev->dma_mem)  in dev_get_coherent_memory()
28 		return dev->dma_mem;  in dev_get_coherent_memory()
33 					     struct dma_coherent_mem * mem)  in dma_get_device_base()  argument
35 	if (mem->use_dev_dma_pfn_offset)  in dma_get_device_base()
36 		return phys_to_dma(dev, PFN_PHYS(mem->pfn_base));  in dma_get_device_base()
37 	return mem->device_base;  in dma_get_device_base()
42 		struct dma_coherent_mem **mem)  in dma_init_coherent_memory()  argument
51 		ret = -EINVAL;  in dma_init_coherent_memory()
57 		ret = -EINVAL;  in dma_init_coherent_memory()
62 		ret = -ENOMEM;  in dma_init_coherent_memory()
65 	dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);  in dma_init_coherent_memory()
66 	if (!dma_mem->bitmap) {  in dma_init_coherent_memory()
67 		ret = -ENOMEM;  in dma_init_coherent_memory()
71 	dma_mem->virt_base = mem_base;  in dma_init_coherent_memory()
72 	dma_mem->device_base = device_addr;  in dma_init_coherent_memory()
73 	dma_mem->pfn_base = PFN_DOWN(phys_addr);  in dma_init_coherent_memory()
74 	dma_mem->size = pages;  in dma_init_coherent_memory()
75 	spin_lock_init(&dma_mem->spinlock);  in dma_init_coherent_memory()
77 	*mem = dma_mem;  in dma_init_coherent_memory()
87 static void dma_release_coherent_memory(struct dma_coherent_mem *mem)  in dma_release_coherent_memory()  argument
89 	if (!mem)  in dma_release_coherent_memory()
92 	memunmap(mem->virt_base);  in dma_release_coherent_memory()
93 	kfree(mem->bitmap);  in dma_release_coherent_memory()
94 	kfree(mem);  in dma_release_coherent_memory()
98 				      struct dma_coherent_mem *mem)  in dma_assign_coherent_memory()  argument
101 		return -ENODEV;  in dma_assign_coherent_memory()
103 	if (dev->dma_mem)  in dma_assign_coherent_memory()
104 		return -EBUSY;  in dma_assign_coherent_memory()
106 	dev->dma_mem = mem;  in dma_assign_coherent_memory()
118  * device_addr is the DMA address the device needs to be programmed with to
130 	struct dma_coherent_mem *mem;  in dma_declare_coherent_memory()  local
133 	ret = dma_init_coherent_memory(phys_addr, device_addr, size, &mem);  in dma_declare_coherent_memory()
137 	ret = dma_assign_coherent_memory(dev, mem);  in dma_declare_coherent_memory()
139 		dma_release_coherent_memory(mem);  in dma_declare_coherent_memory()
144 				       struct dma_coherent_mem *mem,  in __dma_alloc_from_coherent()  argument
152 	spin_lock_irqsave(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
154 	if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT)))  in __dma_alloc_from_coherent()
157 	pageno = bitmap_find_free_region(mem->bitmap, mem->size, order);  in __dma_alloc_from_coherent()
164 	*dma_handle = dma_get_device_base(dev, mem) +  in __dma_alloc_from_coherent()
166 	ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT);  in __dma_alloc_from_coherent()
167 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
171 	spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_alloc_from_coherent()
176  * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
179  * @dma_handle:	This will be filled with the correct dma handle
183  * This function should be only called from per-arch dma_alloc_coherent()
184  * to support allocation from per-device coherent memory pools.
192 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_alloc_from_dev_coherent()  local
194 	if (!mem)  in dma_alloc_from_dev_coherent()
197 	*ret = __dma_alloc_from_coherent(dev, mem, size, dma_handle);  in dma_alloc_from_dev_coherent()
211 static int __dma_release_from_coherent(struct dma_coherent_mem *mem,  in __dma_release_from_coherent()  argument
214 	if (mem && vaddr >= mem->virt_base && vaddr <  in __dma_release_from_coherent()
215 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_release_from_coherent()
216 		int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_release_from_coherent()
219 		spin_lock_irqsave(&mem->spinlock, flags);  in __dma_release_from_coherent()
220 		bitmap_release_region(mem->bitmap, page, order);  in __dma_release_from_coherent()
221 		spin_unlock_irqrestore(&mem->spinlock, flags);  in __dma_release_from_coherent()
228  * dma_release_from_dev_coherent() - free memory to device coherent memory pool
233  * This checks whether the memory was allocated from the per-device
241 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_release_from_dev_coherent()  local
243 	return __dma_release_from_coherent(mem, order, vaddr);  in dma_release_from_dev_coherent()
255 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem,  in __dma_mmap_from_coherent()  argument
258 	if (mem && vaddr >= mem->virt_base && vaddr + size <=  in __dma_mmap_from_coherent()
259 		   (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) {  in __dma_mmap_from_coherent()
260 		unsigned long off = vma->vm_pgoff;  in __dma_mmap_from_coherent()
261 		int start = (vaddr - mem->virt_base) >> PAGE_SHIFT;  in __dma_mmap_from_coherent()
265 		*ret = -ENXIO;  in __dma_mmap_from_coherent()
266 		if (off < count && user_count <= count - off) {  in __dma_mmap_from_coherent()
267 			unsigned long pfn = mem->pfn_base + start + off;  in __dma_mmap_from_coherent()
268 			*ret = remap_pfn_range(vma, vma->vm_start, pfn,  in __dma_mmap_from_coherent()
270 					       vma->vm_page_prot);  in __dma_mmap_from_coherent()
278  * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
285  * This checks whether the memory was allocated from the per-device
295 	struct dma_coherent_mem *mem = dev_get_coherent_memory(dev);  in dma_mmap_from_dev_coherent()  local
297 	return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret);  in dma_mmap_from_dev_coherent()
322 	struct dma_coherent_mem *mem = rmem->priv;  in rmem_dma_device_init()  local
325 	if (!mem) {  in rmem_dma_device_init()
326 		ret = dma_init_coherent_memory(rmem->base, rmem->base,  in rmem_dma_device_init()
327 					       rmem->size, &mem);  in rmem_dma_device_init()
329 			pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n",  in rmem_dma_device_init()
330 				&rmem->base, (unsigned long)rmem->size / SZ_1M);  in rmem_dma_device_init()
334 	mem->use_dev_dma_pfn_offset = true;  in rmem_dma_device_init()
335 	rmem->priv = mem;  in rmem_dma_device_init()
336 	dma_assign_coherent_memory(dev, mem);  in rmem_dma_device_init()
344 		dev->dma_mem = NULL;  in rmem_dma_device_release()
354 	unsigned long node = rmem->fdt_node;  in rmem_dma_setup()
357 		return -EINVAL;  in rmem_dma_setup()
360 	if (!of_get_flat_dt_prop(node, "no-map", NULL)) {  in rmem_dma_setup()
361 		pr_err("Reserved memory: regions without no-map are not yet supported\n");  in rmem_dma_setup()
362 		return -EINVAL;  in rmem_dma_setup()
365 	if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) {  in rmem_dma_setup()
367 		     "Reserved memory: region for default DMA coherent area is redefined\n");  in rmem_dma_setup()
372 	rmem->ops = &rmem_dma_ops;  in rmem_dma_setup()
373 	pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n",  in rmem_dma_setup()
374 		&rmem->base, (unsigned long)rmem->size / SZ_1M);  in rmem_dma_setup()
384 		return -ENOMEM;  in dma_init_reserved_memory()
386 	ops = dma_reserved_default_memory->ops;  in dma_init_reserved_memory()
392 	ret = ops->device_init(dma_reserved_default_memory, NULL);  in dma_init_reserved_memory()
395 		dma_coherent_default_memory = dma_reserved_default_memory->priv;  in dma_init_reserved_memory()
396 		pr_info("DMA: default coherent area is set\n");  in dma_init_reserved_memory()
404 RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);