1 // SPDX-License-Identifier: GPL-2.0-only
9  * This allocator returns small blocks of a given size which are DMA-able by
15  * represented by the 'struct dma_pool' which keeps a doubly-linked list of
17  * least 'size' bytes.  Free blocks are tracked in an unsorted singly-linked
23 #include <linux/dma-mapping.h>
76 	temp = scnprintf(next, size, "poolinfo - 0.1\n");  in pools_show()
77 	size -= temp;  in pools_show()
81 	list_for_each_entry(pool, &dev->dma_pools, pools) {  in pools_show()
85 		spin_lock_irq(&pool->lock);  in pools_show()
86 		list_for_each_entry(page, &pool->page_list, page_list) {  in pools_show()
88 			blocks += page->in_use;  in pools_show()
90 		spin_unlock_irq(&pool->lock);  in pools_show()
92 		/* per-pool info, no real statistics yet */  in pools_show()
93 		temp = scnprintf(next, size, "%-16s %4u %4zu %4zu %2u\n",  in pools_show()
94 				 pool->name, blocks,  in pools_show()
95 				 pages * (pool->allocation / pool->size),  in pools_show()
96 				 pool->size, pages);  in pools_show()
97 		size -= temp;  in pools_show()
102 	return PAGE_SIZE - size;  in pools_show()
108  * dma_pool_create - Creates a pool of consistent memory blocks, for dma.
112  * @align: alignment requirement for blocks; must be a power of two
113  * @boundary: returned blocks won't cross this power of two boundary
139 	else if (align & (align - 1))  in dma_pool_create()
152 	else if ((boundary < size) || (boundary & (boundary - 1)))  in dma_pool_create()
159 	strscpy(retval->name, name, sizeof(retval->name));  in dma_pool_create()
161 	retval->dev = dev;  in dma_pool_create()
163 	INIT_LIST_HEAD(&retval->page_list);  in dma_pool_create()
164 	spin_lock_init(&retval->lock);  in dma_pool_create()
165 	retval->size = size;  in dma_pool_create()
166 	retval->boundary = boundary;  in dma_pool_create()
167 	retval->allocation = allocation;  in dma_pool_create()
169 	INIT_LIST_HEAD(&retval->pools);  in dma_pool_create()
172 	 * pools_lock ensures that the ->dma_pools list does not get corrupted.  in dma_pool_create()
181 	if (list_empty(&dev->dma_pools))  in dma_pool_create()
183 	list_add(&retval->pools, &dev->dma_pools);  in dma_pool_create()
191 			list_del(&retval->pools);  in dma_pool_create()
206 	unsigned int next_boundary = pool->boundary;  in pool_initialise_page()
209 		unsigned int next = offset + pool->size;  in pool_initialise_page()
210 		if (unlikely((next + pool->size) >= next_boundary)) {  in pool_initialise_page()
212 			next_boundary += pool->boundary;  in pool_initialise_page()
214 		*(int *)(page->vaddr + offset) = next;  in pool_initialise_page()
216 	} while (offset < pool->allocation);  in pool_initialise_page()
226 	page->vaddr = dma_alloc_coherent(pool->dev, pool->allocation,  in pool_alloc_page()
227 					 &page->dma, mem_flags);  in pool_alloc_page()
228 	if (page->vaddr) {  in pool_alloc_page()
230 		memset(page->vaddr, POOL_POISON_FREED, pool->allocation);  in pool_alloc_page()
233 		page->in_use = 0;  in pool_alloc_page()
234 		page->offset = 0;  in pool_alloc_page()
244 	return page->in_use != 0;  in is_page_busy()
249 	dma_addr_t dma = page->dma;  in pool_free_page()
252 	memset(page->vaddr, POOL_POISON_FREED, pool->allocation);  in pool_free_page()
254 	dma_free_coherent(pool->dev, pool->allocation, page->vaddr, dma);  in pool_free_page()
255 	list_del(&page->page_list);  in pool_free_page()
260  * dma_pool_destroy - destroys a pool of dma memory blocks.
277 	list_del(&pool->pools);  in dma_pool_destroy()
278 	if (pool->dev && list_empty(&pool->dev->dma_pools))  in dma_pool_destroy()
282 		device_remove_file(pool->dev, &dev_attr_pools);  in dma_pool_destroy()
285 	list_for_each_entry_safe(page, tmp, &pool->page_list, page_list) {  in dma_pool_destroy()
287 			if (pool->dev)  in dma_pool_destroy()
288 				dev_err(pool->dev, "%s %s, %p busy\n", __func__,  in dma_pool_destroy()
289 					pool->name, page->vaddr);  in dma_pool_destroy()
292 				       pool->name, page->vaddr);  in dma_pool_destroy()
293 			/* leak the still-in-use consistent memory */  in dma_pool_destroy()
294 			list_del(&page->page_list);  in dma_pool_destroy()
305  * dma_pool_alloc - get a block of consistent memory
324 	spin_lock_irqsave(&pool->lock, flags);  in dma_pool_alloc()
325 	list_for_each_entry(page, &pool->page_list, page_list) {  in dma_pool_alloc()
326 		if (page->offset < pool->allocation)  in dma_pool_alloc()
330 	/* pool_alloc_page() might sleep, so temporarily drop &pool->lock */  in dma_pool_alloc()
331 	spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_alloc()
337 	spin_lock_irqsave(&pool->lock, flags);  in dma_pool_alloc()
339 	list_add(&page->page_list, &pool->page_list);  in dma_pool_alloc()
341 	page->in_use++;  in dma_pool_alloc()
342 	offset = page->offset;  in dma_pool_alloc()
343 	page->offset = *(int *)(page->vaddr + offset);  in dma_pool_alloc()
344 	retval = offset + page->vaddr;  in dma_pool_alloc()
345 	*handle = offset + page->dma;  in dma_pool_alloc()
350 		/* page->offset is stored in first 4 bytes */  in dma_pool_alloc()
351 		for (i = sizeof(page->offset); i < pool->size; i++) {  in dma_pool_alloc()
354 			if (pool->dev)  in dma_pool_alloc()
355 				dev_err(pool->dev, "%s %s, %p (corrupted)\n",  in dma_pool_alloc()
356 					__func__, pool->name, retval);  in dma_pool_alloc()
359 					__func__, pool->name, retval);  in dma_pool_alloc()
366 					data, pool->size, 1);  in dma_pool_alloc()
371 		memset(retval, POOL_POISON_ALLOCATED, pool->size);  in dma_pool_alloc()
373 	spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_alloc()
376 		memset(retval, 0, pool->size);  in dma_pool_alloc()
386 	list_for_each_entry(page, &pool->page_list, page_list) {  in pool_find_page()
387 		if (dma < page->dma)  in pool_find_page()
389 		if ((dma - page->dma) < pool->allocation)  in pool_find_page()
396  * dma_pool_free - put block back into dma pool
402  * unless it is first re-allocated.
410 	spin_lock_irqsave(&pool->lock, flags);  in dma_pool_free()
413 		spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_free()
414 		if (pool->dev)  in dma_pool_free()
415 			dev_err(pool->dev, "%s %s, %p/%pad (bad dma)\n",  in dma_pool_free()
416 				__func__, pool->name, vaddr, &dma);  in dma_pool_free()
419 			       __func__, pool->name, vaddr, &dma);  in dma_pool_free()
423 	offset = vaddr - page->vaddr;  in dma_pool_free()
425 		memset(vaddr, 0, pool->size);  in dma_pool_free()
427 	if ((dma - page->dma) != offset) {  in dma_pool_free()
428 		spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_free()
429 		if (pool->dev)  in dma_pool_free()
430 			dev_err(pool->dev, "%s %s, %p (bad vaddr)/%pad\n",  in dma_pool_free()
431 				__func__, pool->name, vaddr, &dma);  in dma_pool_free()
434 			       __func__, pool->name, vaddr, &dma);  in dma_pool_free()
438 		unsigned int chain = page->offset;  in dma_pool_free()
439 		while (chain < pool->allocation) {  in dma_pool_free()
441 				chain = *(int *)(page->vaddr + chain);  in dma_pool_free()
444 			spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_free()
445 			if (pool->dev)  in dma_pool_free()
446 				dev_err(pool->dev, "%s %s, dma %pad already free\n",  in dma_pool_free()
447 					__func__, pool->name, &dma);  in dma_pool_free()
450 				       __func__, pool->name, &dma);  in dma_pool_free()
454 	memset(vaddr, POOL_POISON_FREED, pool->size);  in dma_pool_free()
457 	page->in_use--;  in dma_pool_free()
458 	*(int *)vaddr = page->offset;  in dma_pool_free()
459 	page->offset = offset;  in dma_pool_free()
465 	spin_unlock_irqrestore(&pool->lock, flags);  in dma_pool_free()
470  * Managed DMA pool
485  * dmam_pool_create - Managed dma_pool_create()
489  * @align: alignment requirement for blocks; must be a power of two
492  * Managed dma_pool_create().  DMA pool created with this function is
495  * Return: a managed dma allocation pool with the requested
518  * dmam_pool_destroy - Managed dma_pool_destroy()
521  * Managed dma_pool_destroy().
525 	struct device *dev = pool->dev;  in dmam_pool_destroy()