1 // SPDX-License-Identifier: GPL-2.0
8 #include <linux/dma-map-ops.h>
9 #include <linux/dma-direct.h>
26 /* Dynamic background expansion when the atomic pool is near capacity */
71 	end = cma_get_base(cma) + size - 1;  in cma_in_zone()
79 static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size,  in atomic_pool_expand()  argument
85 	int ret = -ENOMEM;  in atomic_pool_expand()
87 	/* Cannot allocate larger than MAX_ORDER-1 */  in atomic_pool_expand()
88 	order = min(get_order(pool_size), MAX_ORDER-1);  in atomic_pool_expand()
97 	} while (!page && order-- > 0);  in atomic_pool_expand()
113 	 * Memory in the atomic DMA pools must be unencrypted, the pools do not  in atomic_pool_expand()
114 	 * shrink so no re-encryption occurs in dma_direct_free().  in atomic_pool_expand()
120 	ret = gen_pool_add_virt(pool, (unsigned long)addr, page_to_phys(page),  in atomic_pool_expand()
145 static void atomic_pool_resize(struct gen_pool *pool, gfp_t gfp)  in atomic_pool_resize()  argument
147 	if (pool && gen_pool_avail(pool) < atomic_pool_size)  in atomic_pool_resize()
148 		atomic_pool_expand(pool, gen_pool_size(pool), gfp);  in atomic_pool_resize()
165 	struct gen_pool *pool;  in __dma_atomic_pool_init()  local
168 	pool = gen_pool_create(PAGE_SHIFT, NUMA_NO_NODE);  in __dma_atomic_pool_init()
169 	if (!pool)  in __dma_atomic_pool_init()
172 	gen_pool_set_algo(pool, gen_pool_first_fit_order_align, NULL);  in __dma_atomic_pool_init()
174 	ret = atomic_pool_expand(pool, pool_size, gfp);  in __dma_atomic_pool_init()
176 		gen_pool_destroy(pool);  in __dma_atomic_pool_init()
177 		pr_err("DMA: failed to allocate %zu KiB %pGg pool for atomic allocation\n",  in __dma_atomic_pool_init()
182 	pr_info("DMA: preallocated %zu KiB %pGg pool for atomic allocations\n",  in __dma_atomic_pool_init()
183 		gen_pool_size(pool) >> 10, &gfp);  in __dma_atomic_pool_init()
184 	return pool;  in __dma_atomic_pool_init()
192 	 * If coherent_pool was not used on the command line, default the pool  in dma_atomic_pool_init()
193 	 * sizes to 128KB per 1GB of memory, min 128KB, max MAX_ORDER-1.  in dma_atomic_pool_init()
205 		ret = -ENOMEM;  in dma_atomic_pool_init()
210 			ret = -ENOMEM;  in dma_atomic_pool_init()
216 			ret = -ENOMEM;  in dma_atomic_pool_init()
241 		struct gen_pool *pool, void **cpu_addr,  in __dma_alloc_from_pool()  argument
247 	addr = gen_pool_alloc(pool, size);  in __dma_alloc_from_pool()
251 	phys = gen_pool_virt_to_phys(pool, addr);  in __dma_alloc_from_pool()
253 		gen_pool_free(pool, addr, size);  in __dma_alloc_from_pool()
257 	if (gen_pool_avail(pool) < atomic_pool_size)  in __dma_alloc_from_pool()
269 	struct gen_pool *pool = NULL;  in dma_alloc_from_pool()  local
272 	while ((pool = dma_guess_pool(pool, gfp))) {  in dma_alloc_from_pool()
273 		page = __dma_alloc_from_pool(dev, size, pool, cpu_addr,  in dma_alloc_from_pool()
279 	WARN(1, "Failed to get suitable pool for %s\n", dev_name(dev));  in dma_alloc_from_pool()
285 	struct gen_pool *pool = NULL;  in dma_free_from_pool()  local
287 	while ((pool = dma_guess_pool(pool, 0))) {  in dma_free_from_pool()
288 		if (!gen_pool_has_addr(pool, (unsigned long)start, size))  in dma_free_from_pool()
290 		gen_pool_free(pool, (unsigned long)start, size);  in dma_free_from_pool()