Lines Matching +full:dma +full:- +full:mem
1 // SPDX-License-Identifier: GPL-2.0
3 * Coherent per-device memory handling.
10 #include <linux/dma-direct.h>
11 #include <linux/dma-map-ops.h>
27 if (dev && dev->dma_mem) in dev_get_coherent_memory()
28 return dev->dma_mem; in dev_get_coherent_memory()
33 struct dma_coherent_mem * mem) in dma_get_device_base() argument
35 if (mem->use_dev_dma_pfn_offset) in dma_get_device_base()
36 return phys_to_dma(dev, PFN_PHYS(mem->pfn_base)); in dma_get_device_base()
37 return mem->device_base; in dma_get_device_base()
42 struct dma_coherent_mem **mem) in dma_init_coherent_memory() argument
51 ret = -EINVAL; in dma_init_coherent_memory()
57 ret = -EINVAL; in dma_init_coherent_memory()
62 ret = -ENOMEM; in dma_init_coherent_memory()
65 dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL); in dma_init_coherent_memory()
66 if (!dma_mem->bitmap) { in dma_init_coherent_memory()
67 ret = -ENOMEM; in dma_init_coherent_memory()
71 dma_mem->virt_base = mem_base; in dma_init_coherent_memory()
72 dma_mem->device_base = device_addr; in dma_init_coherent_memory()
73 dma_mem->pfn_base = PFN_DOWN(phys_addr); in dma_init_coherent_memory()
74 dma_mem->size = pages; in dma_init_coherent_memory()
75 spin_lock_init(&dma_mem->spinlock); in dma_init_coherent_memory()
77 *mem = dma_mem; in dma_init_coherent_memory()
87 static void dma_release_coherent_memory(struct dma_coherent_mem *mem) in dma_release_coherent_memory() argument
89 if (!mem) in dma_release_coherent_memory()
92 memunmap(mem->virt_base); in dma_release_coherent_memory()
93 kfree(mem->bitmap); in dma_release_coherent_memory()
94 kfree(mem); in dma_release_coherent_memory()
98 struct dma_coherent_mem *mem) in dma_assign_coherent_memory() argument
101 return -ENODEV; in dma_assign_coherent_memory()
103 if (dev->dma_mem) in dma_assign_coherent_memory()
104 return -EBUSY; in dma_assign_coherent_memory()
106 dev->dma_mem = mem; in dma_assign_coherent_memory()
118 * device_addr is the DMA address the device needs to be programmed with to
130 struct dma_coherent_mem *mem; in dma_declare_coherent_memory() local
133 ret = dma_init_coherent_memory(phys_addr, device_addr, size, &mem); in dma_declare_coherent_memory()
137 ret = dma_assign_coherent_memory(dev, mem); in dma_declare_coherent_memory()
139 dma_release_coherent_memory(mem); in dma_declare_coherent_memory()
144 struct dma_coherent_mem *mem, in __dma_alloc_from_coherent() argument
152 spin_lock_irqsave(&mem->spinlock, flags); in __dma_alloc_from_coherent()
154 if (unlikely(size > ((dma_addr_t)mem->size << PAGE_SHIFT))) in __dma_alloc_from_coherent()
157 pageno = bitmap_find_free_region(mem->bitmap, mem->size, order); in __dma_alloc_from_coherent()
164 *dma_handle = dma_get_device_base(dev, mem) + in __dma_alloc_from_coherent()
166 ret = mem->virt_base + ((dma_addr_t)pageno << PAGE_SHIFT); in __dma_alloc_from_coherent()
167 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
171 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_alloc_from_coherent()
176 * dma_alloc_from_dev_coherent() - allocate memory from device coherent pool
179 * @dma_handle: This will be filled with the correct dma handle
183 * This function should be only called from per-arch dma_alloc_coherent()
184 * to support allocation from per-device coherent memory pools.
192 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_alloc_from_dev_coherent() local
194 if (!mem) in dma_alloc_from_dev_coherent()
197 *ret = __dma_alloc_from_coherent(dev, mem, size, dma_handle); in dma_alloc_from_dev_coherent()
211 static int __dma_release_from_coherent(struct dma_coherent_mem *mem, in __dma_release_from_coherent() argument
214 if (mem && vaddr >= mem->virt_base && vaddr < in __dma_release_from_coherent()
215 (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { in __dma_release_from_coherent()
216 int page = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_release_from_coherent()
219 spin_lock_irqsave(&mem->spinlock, flags); in __dma_release_from_coherent()
220 bitmap_release_region(mem->bitmap, page, order); in __dma_release_from_coherent()
221 spin_unlock_irqrestore(&mem->spinlock, flags); in __dma_release_from_coherent()
228 * dma_release_from_dev_coherent() - free memory to device coherent memory pool
233 * This checks whether the memory was allocated from the per-device
241 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_release_from_dev_coherent() local
243 return __dma_release_from_coherent(mem, order, vaddr); in dma_release_from_dev_coherent()
255 static int __dma_mmap_from_coherent(struct dma_coherent_mem *mem, in __dma_mmap_from_coherent() argument
258 if (mem && vaddr >= mem->virt_base && vaddr + size <= in __dma_mmap_from_coherent()
259 (mem->virt_base + ((dma_addr_t)mem->size << PAGE_SHIFT))) { in __dma_mmap_from_coherent()
260 unsigned long off = vma->vm_pgoff; in __dma_mmap_from_coherent()
261 int start = (vaddr - mem->virt_base) >> PAGE_SHIFT; in __dma_mmap_from_coherent()
265 *ret = -ENXIO; in __dma_mmap_from_coherent()
266 if (off < count && user_count <= count - off) { in __dma_mmap_from_coherent()
267 unsigned long pfn = mem->pfn_base + start + off; in __dma_mmap_from_coherent()
268 *ret = remap_pfn_range(vma, vma->vm_start, pfn, in __dma_mmap_from_coherent()
270 vma->vm_page_prot); in __dma_mmap_from_coherent()
278 * dma_mmap_from_dev_coherent() - mmap memory from the device coherent pool
285 * This checks whether the memory was allocated from the per-device
295 struct dma_coherent_mem *mem = dev_get_coherent_memory(dev); in dma_mmap_from_dev_coherent() local
297 return __dma_mmap_from_coherent(mem, vma, vaddr, size, ret); in dma_mmap_from_dev_coherent()
322 struct dma_coherent_mem *mem = rmem->priv; in rmem_dma_device_init() local
325 if (!mem) { in rmem_dma_device_init()
326 ret = dma_init_coherent_memory(rmem->base, rmem->base, in rmem_dma_device_init()
327 rmem->size, &mem); in rmem_dma_device_init()
329 pr_err("Reserved memory: failed to init DMA memory pool at %pa, size %ld MiB\n", in rmem_dma_device_init()
330 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_dma_device_init()
334 mem->use_dev_dma_pfn_offset = true; in rmem_dma_device_init()
335 rmem->priv = mem; in rmem_dma_device_init()
336 dma_assign_coherent_memory(dev, mem); in rmem_dma_device_init()
344 dev->dma_mem = NULL; in rmem_dma_device_release()
354 unsigned long node = rmem->fdt_node; in rmem_dma_setup()
357 return -EINVAL; in rmem_dma_setup()
360 if (!of_get_flat_dt_prop(node, "no-map", NULL)) { in rmem_dma_setup()
361 pr_err("Reserved memory: regions without no-map are not yet supported\n"); in rmem_dma_setup()
362 return -EINVAL; in rmem_dma_setup()
365 if (of_get_flat_dt_prop(node, "linux,dma-default", NULL)) { in rmem_dma_setup()
367 "Reserved memory: region for default DMA coherent area is redefined\n"); in rmem_dma_setup()
372 rmem->ops = &rmem_dma_ops; in rmem_dma_setup()
373 pr_info("Reserved memory: created DMA memory pool at %pa, size %ld MiB\n", in rmem_dma_setup()
374 &rmem->base, (unsigned long)rmem->size / SZ_1M); in rmem_dma_setup()
384 return -ENOMEM; in dma_init_reserved_memory()
386 ops = dma_reserved_default_memory->ops; in dma_init_reserved_memory()
392 ret = ops->device_init(dma_reserved_default_memory, NULL); in dma_init_reserved_memory()
395 dma_coherent_default_memory = dma_reserved_default_memory->priv; in dma_init_reserved_memory()
396 pr_info("DMA: default coherent area is set\n"); in dma_init_reserved_memory()
404 RESERVEDMEM_OF_DECLARE(dma, "shared-dma-pool", rmem_dma_setup);