1 // SPDX-License-Identifier: GPL-2.0
3  * arch-independent dma-mapping routines
10 #include <linux/dma-map-ops.h>
23  * Managed DMA API
36 	dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,  in dmam_release()
37 			this->attrs);  in dmam_release()
44 	if (this->vaddr == match->vaddr) {  in dmam_match()
45 		WARN_ON(this->size != match->size ||  in dmam_match()
46 			this->dma_handle != match->dma_handle);  in dmam_match()
53  * dmam_free_coherent - Managed dma_free_coherent()
54  * @dev: Device to free coherent memory for
57  * @dma_handle: DMA handle of the memory to free
72  * dmam_alloc_attrs - Managed dma_alloc_attrs()
75  * @dma_handle: Out argument for allocated DMA handle
101 	dr->vaddr = vaddr;  in dmam_alloc_attrs()
102 	dr->dma_handle = *dma_handle;  in dmam_alloc_attrs()
103 	dr->size = size;  in dmam_alloc_attrs()
104 	dr->attrs = attrs;  in dmam_alloc_attrs()
118 	if (dev->dma_ops_bypass)  in dma_go_direct()
119 		return min_not_zero(mask, dev->bus_dma_limit) >=  in dma_go_direct()
127  * Check if the devices uses a direct mapping for streaming DMA operations.
128  * This allows IOMMU drivers to set a bypass mode if the DMA mask is large
134 	return dma_go_direct(dev, dev->coherent_dma_mask, ops);  in dma_alloc_direct()
140 	return dma_go_direct(dev, *dev->dma_mask, ops);  in dma_map_direct()
152 	if (WARN_ON_ONCE(!dev->dma_mask))  in dma_map_page_attrs()
159 		addr = ops->map_page(dev, page, offset, size, dir, attrs);  in dma_map_page_attrs()
176 	else if (ops->unmap_page)  in dma_unmap_page_attrs()
177 		ops->unmap_page(dev, addr, size, dir, attrs);  in dma_unmap_page_attrs()
190 	if (WARN_ON_ONCE(!dev->dma_mask))  in __dma_map_sg_attrs()
197 		ents = ops->map_sg(dev, sg, nents, dir, attrs);  in __dma_map_sg_attrs()
202 	} else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM &&  in __dma_map_sg_attrs()
203 				ents != -EIO && ents != -EREMOTEIO)) {  in __dma_map_sg_attrs()
204 		return -EIO;  in __dma_map_sg_attrs()
211  * dma_map_sg_attrs - Map the given buffer for DMA
212  * @dev:	The device for which to perform the DMA operation
215  * @dir:	DMA direction
216  * @attrs:	Optional DMA attributes for the map operation
219  * nents segments for the @dir DMA operation by the @dev device.
240  * dma_map_sgtable - Map the given buffer for DMA
241  * @dev:	The device for which to perform the DMA operation
243  * @dir:	DMA direction
244  * @attrs:	Optional DMA attributes for the map operation
247  * object for the @dir DMA operation by the @dev device. After success, the
248  * ownership for the buffer is transferred to the DMA domain.  One has to
256  *   -EINVAL		An invalid argument, unaligned access or other error
258  *   -ENOMEM		Insufficient resources (like memory or IOVA space) to
260  *   -EIO		Legacy error code with an unknown meaning. eg. this is
263  *   -EREMOTEIO		The DMA device cannot access P2PDMA memory specified
271 	nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);  in dma_map_sgtable()
274 	sgt->nents = nents;  in dma_map_sgtable()
290 	else if (ops->unmap_sg)  in dma_unmap_sg_attrs()
291 		ops->unmap_sg(dev, sg, nents, dir, attrs);  in dma_unmap_sg_attrs()
303 	if (WARN_ON_ONCE(!dev->dma_mask))  in dma_map_resource()
308 	else if (ops->map_resource)  in dma_map_resource()
309 		addr = ops->map_resource(dev, phys_addr, size, dir, attrs);  in dma_map_resource()
322 	if (!dma_map_direct(dev, ops) && ops->unmap_resource)  in dma_unmap_resource()
323 		ops->unmap_resource(dev, addr, size, dir, attrs);  in dma_unmap_resource()
336 	else if (ops->sync_single_for_cpu)  in dma_sync_single_for_cpu()
337 		ops->sync_single_for_cpu(dev, addr, size, dir);  in dma_sync_single_for_cpu()
350 	else if (ops->sync_single_for_device)  in dma_sync_single_for_device()
351 		ops->sync_single_for_device(dev, addr, size, dir);  in dma_sync_single_for_device()
364 	else if (ops->sync_sg_for_cpu)  in dma_sync_sg_for_cpu()
365 		ops->sync_sg_for_cpu(dev, sg, nelems, dir);  in dma_sync_sg_for_cpu()
378 	else if (ops->sync_sg_for_device)  in dma_sync_sg_for_device()
379 		ops->sync_sg_for_device(dev, sg, nelems, dir);  in dma_sync_sg_for_device()
385  * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems
387  * coherent DMA APIs through the dma_buf API, which only accepts a
389  * 1. Not all memory allocated via the coherent DMA APIs is backed by
391  * 2. Passing coherent DMA memory into the streaming APIs is not allowed
404 	if (!ops->get_sgtable)  in dma_get_sgtable_attrs()
405 		return -ENXIO;  in dma_get_sgtable_attrs()
406 	return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);  in dma_get_sgtable_attrs()
428  * dma_can_mmap - check if a given device supports dma_mmap_*
432  * map DMA allocations to userspace.
440 	return ops->mmap != NULL;  in dma_can_mmap()
445  * dma_mmap_attrs - map a coherent DMA allocation into user space
446  * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
448  * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
449  * @dma_addr: device-view address returned from dma_alloc_attrs
453  * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user
454  * space.  The coherent DMA buffer must not be freed by the driver until the
466 	if (!ops->mmap)  in dma_mmap_attrs()
467 		return -ENXIO;  in dma_mmap_attrs()
468 	return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);  in dma_mmap_attrs()
478 	if (ops->get_required_mask)  in dma_get_required_mask()
479 		return ops->get_required_mask(dev);  in dma_get_required_mask()
482 	 * We require every DMA ops implementation to at least support a 32-bit  in dma_get_required_mask()
483 	 * DMA mask (and use bounce buffering if that isn't supported in  in dma_get_required_mask()
485 	 * actually report an optimal mask we default to 32-bit here as that  in dma_get_required_mask()
499 	WARN_ON_ONCE(!dev->coherent_dma_mask);  in dma_alloc_attrs()
509 	else if (ops->alloc)  in dma_alloc_attrs()
510 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
527 	 * On non-coherent platforms which implement DMA-coherent buffers via  in dma_free_attrs()
528 	 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting  in dma_free_attrs()
531 	 * probably misusing the coherent API anyway.  in dma_free_attrs()
541 	else if (ops->free)  in dma_free_attrs()
542 		ops->free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
551 	if (WARN_ON_ONCE(!dev->coherent_dma_mask))  in __dma_alloc_pages()
559 	if (!ops->alloc_pages)  in __dma_alloc_pages()
561 	return ops->alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
583 	else if (ops->free_pages)  in __dma_free_pages()
584 		ops->free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
600 	if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff)  in dma_mmap_pages()
601 		return -ENXIO;  in dma_mmap_pages()
602 	return remap_pfn_range(vma, vma->vm_start,  in dma_mmap_pages()
603 			       page_to_pfn(page) + vma->vm_pgoff,  in dma_mmap_pages()
604 			       vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot);  in dma_mmap_pages()
619 	page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp);  in alloc_single_sgt()
622 	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in alloc_single_sgt()
623 	sg_dma_len(sgt->sgl) = sgt->sgl->length;  in alloc_single_sgt()
641 	if (ops && ops->alloc_noncontiguous)  in dma_alloc_noncontiguous()
642 		sgt = ops->alloc_noncontiguous(dev, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
647 		sgt->nents = 1;  in dma_alloc_noncontiguous()
648 		debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs);  in dma_alloc_noncontiguous()
657 	__dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address,  in free_single_sgt()
668 	debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir);  in dma_free_noncontiguous()
669 	if (ops && ops->free_noncontiguous)  in dma_free_noncontiguous()
670 		ops->free_noncontiguous(dev, size, sgt, dir);  in dma_free_noncontiguous()
682 	if (ops && ops->alloc_noncontiguous)  in dma_vmap_noncontiguous()
683 		return vmap(sgt_handle(sgt)->pages, count, VM_MAP, PAGE_KERNEL);  in dma_vmap_noncontiguous()
684 	return page_address(sg_page(sgt->sgl));  in dma_vmap_noncontiguous()
692 	if (ops && ops->alloc_noncontiguous)  in dma_vunmap_noncontiguous()
702 	if (ops && ops->alloc_noncontiguous) {  in dma_mmap_noncontiguous()
705 		if (vma->vm_pgoff >= count ||  in dma_mmap_noncontiguous()
706 		    vma_pages(vma) > count - vma->vm_pgoff)  in dma_mmap_noncontiguous()
707 			return -ENXIO;  in dma_mmap_noncontiguous()
708 		return vm_map_pages(vma, sgt_handle(sgt)->pages, count);  in dma_mmap_noncontiguous()
710 	return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl));  in dma_mmap_noncontiguous()
719 	 * ->dma_supported sets the bypass flag, so we must always call  in dma_supported()
724 	if (!ops->dma_supported)  in dma_supported()
726 	return ops->dma_supported(dev, mask);  in dma_supported()
733 	/* if ops is not set, dma direct will be used which supports P2PDMA */  in dma_pci_p2pdma_supported()
739 	 * not be used with dma mapping ops that do not have support even  in dma_pci_p2pdma_supported()
743 	return ops->flags & DMA_F_PCI_P2PDMA_SUPPORTED;  in dma_pci_p2pdma_supported()
761 	if (!dev->dma_mask || !dma_supported(dev, mask))  in dma_set_mask()
762 		return -EIO;  in dma_set_mask()
765 	*dev->dma_mask = mask;  in dma_set_mask()
779 		return -EIO;  in dma_set_coherent_mask()
781 	dev->coherent_dma_mask = mask;  in dma_set_coherent_mask()
793 	else if (ops && ops->max_mapping_size)  in dma_max_mapping_size()
794 		size = ops->max_mapping_size(dev);  in dma_max_mapping_size()
805 	if (ops && ops->opt_mapping_size)  in dma_opt_mapping_size()
806 		size = ops->opt_mapping_size();  in dma_opt_mapping_size()
818 	return ops->sync_single_for_cpu || ops->sync_single_for_device;  in dma_need_sync()
826 	if (!ops || !ops->get_merge_boundary)  in dma_get_merge_boundary()
829 	return ops->get_merge_boundary(dev);  in dma_get_merge_boundary()