1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (C) 2018-2020 Christoph Hellwig.
10 #include <linux/dma-map-ops.h>
21  * override the variable below for dma-direct to work properly.
33 static inline struct page *dma_direct_to_page(struct device *dev,  in dma_direct_to_page()
41 	phys_addr_t phys = (phys_addr_t)(max_pfn - 1) << PAGE_SHIFT;  in dma_direct_get_required_mask()
44 	return (1ULL << (fls64(max_dma) - 1)) * 2 - 1;  in dma_direct_get_required_mask()
50 	u64 dma_limit = min_not_zero(dma_mask, dev->bus_dma_limit);  in dma_direct_optimal_gfp_mask()
68 static bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size)  in dma_coherent_ok()  argument
74 	return dma_addr + size - 1 <=  in dma_coherent_ok()
75 		min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit);  in dma_coherent_ok()
78 static int dma_set_decrypted(struct device *dev, void *vaddr, size_t size)  in dma_set_decrypted()  argument
82 	return set_memory_decrypted((unsigned long)vaddr, PFN_UP(size));  in dma_set_decrypted()
85 static int dma_set_encrypted(struct device *dev, void *vaddr, size_t size)  in dma_set_encrypted()  argument
91 	ret = set_memory_encrypted((unsigned long)vaddr, PFN_UP(size));  in dma_set_encrypted()
93 		pr_warn_ratelimited("leaking DMA memory that can't be re-encrypted\n");  in dma_set_encrypted()
97 static void __dma_direct_free_pages(struct device *dev, struct page *page,  in __dma_direct_free_pages()  argument
98 				    size_t size)  in __dma_direct_free_pages()  argument
100 	if (swiotlb_free(dev, page, size))  in __dma_direct_free_pages()
102 	dma_free_contiguous(dev, page, size);  in __dma_direct_free_pages()
105 static struct page *dma_direct_alloc_swiotlb(struct device *dev, size_t size)  in dma_direct_alloc_swiotlb()  argument
107 	struct page *page = swiotlb_alloc(dev, size);  in dma_direct_alloc_swiotlb()  local
109 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in dma_direct_alloc_swiotlb()
110 		swiotlb_free(dev, page, size);  in dma_direct_alloc_swiotlb()
114 	return page;  in dma_direct_alloc_swiotlb()
117 static struct page *__dma_direct_alloc_pages(struct device *dev, size_t size,  in __dma_direct_alloc_pages()  argument
121 	struct page *page = NULL;  in __dma_direct_alloc_pages()  local
124 	WARN_ON_ONCE(!PAGE_ALIGNED(size));  in __dma_direct_alloc_pages()
127 		return dma_direct_alloc_swiotlb(dev, size);  in __dma_direct_alloc_pages()
129 	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,  in __dma_direct_alloc_pages()
131 	page = dma_alloc_contiguous(dev, size, gfp);  in __dma_direct_alloc_pages()
132 	if (page) {  in __dma_direct_alloc_pages()
133 		if (!dma_coherent_ok(dev, page_to_phys(page), size) ||  in __dma_direct_alloc_pages()
134 		    (!allow_highmem && PageHighMem(page))) {  in __dma_direct_alloc_pages()
135 			dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
136 			page = NULL;  in __dma_direct_alloc_pages()
140 	if (!page)  in __dma_direct_alloc_pages()
141 		page = alloc_pages_node(node, gfp, get_order(size));  in __dma_direct_alloc_pages()
142 	if (page && !dma_coherent_ok(dev, page_to_phys(page), size)) {  in __dma_direct_alloc_pages()
143 		dma_free_contiguous(dev, page, size);  in __dma_direct_alloc_pages()
144 		page = NULL;  in __dma_direct_alloc_pages()
159 	return page;  in __dma_direct_alloc_pages()
171 static void *dma_direct_alloc_from_pool(struct device *dev, size_t size,  in dma_direct_alloc_from_pool()  argument
174 	struct page *page;  in dma_direct_alloc_from_pool()  local
181 	gfp |= dma_direct_optimal_gfp_mask(dev, dev->coherent_dma_mask,  in dma_direct_alloc_from_pool()
183 	page = dma_alloc_from_pool(dev, size, &ret, gfp, dma_coherent_ok);  in dma_direct_alloc_from_pool()
184 	if (!page)  in dma_direct_alloc_from_pool()
186 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc_from_pool()
190 static void *dma_direct_alloc_no_mapping(struct device *dev, size_t size,  in dma_direct_alloc_no_mapping()  argument
193 	struct page *page;  in dma_direct_alloc_no_mapping()  local
195 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);  in dma_direct_alloc_no_mapping()
196 	if (!page)  in dma_direct_alloc_no_mapping()
200 	if (!PageHighMem(page))  in dma_direct_alloc_no_mapping()
201 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc_no_mapping()
203 	/* return the page pointer as the opaque cookie */  in dma_direct_alloc_no_mapping()
204 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc_no_mapping()
205 	return page;  in dma_direct_alloc_no_mapping()
208 void *dma_direct_alloc(struct device *dev, size_t size,  in dma_direct_alloc()  argument
212 	struct page *page;  in dma_direct_alloc()  local
215 	size = PAGE_ALIGN(size);  in dma_direct_alloc()
221 		return dma_direct_alloc_no_mapping(dev, size, dma_handle, gfp);  in dma_direct_alloc()
232 			return arch_dma_alloc(dev, size, dma_handle, gfp,  in dma_direct_alloc()
237 		 * non-coherent devices.  in dma_direct_alloc()
240 			return dma_alloc_from_global_coherent(dev, size,  in dma_direct_alloc()
251 				return dma_direct_alloc_from_pool(dev, size,  in dma_direct_alloc()
265 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);  in dma_direct_alloc()
268 	page = __dma_direct_alloc_pages(dev, size, gfp & ~__GFP_ZERO, true);  in dma_direct_alloc()
269 	if (!page)  in dma_direct_alloc()
274 	 * combination the cma= arguments and per-arch setup.  These need to be  in dma_direct_alloc()
277 	if (PageHighMem(page)) {  in dma_direct_alloc()
289 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
292 		ret = dma_common_contiguous_remap(page, size, prot,  in dma_direct_alloc()
297 		ret = page_address(page);  in dma_direct_alloc()
298 		if (dma_set_decrypted(dev, ret, size))  in dma_direct_alloc()
302 	memset(ret, 0, size);  in dma_direct_alloc()
305 		arch_dma_prep_coherent(page, size);  in dma_direct_alloc()
306 		ret = arch_dma_set_uncached(ret, size);  in dma_direct_alloc()
311 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc()
315 	if (dma_set_encrypted(dev, page_address(page), size))  in dma_direct_alloc()
318 	__dma_direct_free_pages(dev, page, size);  in dma_direct_alloc()
322 void dma_direct_free(struct device *dev, size_t size,  in dma_direct_free()  argument
325 	unsigned int page_order = get_order(size);  in dma_direct_free()
329 		/* cpu_addr is a struct page cookie, not a kernel address */  in dma_direct_free()
330 		dma_free_contiguous(dev, cpu_addr, size);  in dma_direct_free()
339 		arch_dma_free(dev, size, cpu_addr, dma_addr, attrs);  in dma_direct_free()
352 	    dma_free_from_pool(dev, cpu_addr, PAGE_ALIGN(size)))  in dma_direct_free()
359 			arch_dma_clear_uncached(cpu_addr, size);  in dma_direct_free()
360 		if (dma_set_encrypted(dev, cpu_addr, size))  in dma_direct_free()
364 	__dma_direct_free_pages(dev, dma_direct_to_page(dev, dma_addr), size);  in dma_direct_free()
367 struct page *dma_direct_alloc_pages(struct device *dev, size_t size,  in dma_direct_alloc_pages()  argument
370 	struct page *page;  in dma_direct_alloc_pages()  local
374 		return dma_direct_alloc_from_pool(dev, size, dma_handle, gfp);  in dma_direct_alloc_pages()
376 	page = __dma_direct_alloc_pages(dev, size, gfp, false);  in dma_direct_alloc_pages()
377 	if (!page)  in dma_direct_alloc_pages()
380 	ret = page_address(page);  in dma_direct_alloc_pages()
381 	if (dma_set_decrypted(dev, ret, size))  in dma_direct_alloc_pages()
383 	memset(ret, 0, size);  in dma_direct_alloc_pages()
384 	*dma_handle = phys_to_dma_direct(dev, page_to_phys(page));  in dma_direct_alloc_pages()
385 	return page;  in dma_direct_alloc_pages()
387 	__dma_direct_free_pages(dev, page, size);  in dma_direct_alloc_pages()
391 void dma_direct_free_pages(struct device *dev, size_t size,  in dma_direct_free_pages()  argument
392 		struct page *page, dma_addr_t dma_addr,  in dma_direct_free_pages()  argument
395 	void *vaddr = page_address(page);  in dma_direct_free_pages()
399 	    dma_free_from_pool(dev, vaddr, size))  in dma_direct_free_pages()
402 	if (dma_set_encrypted(dev, vaddr, size))  in dma_direct_free_pages()
404 	__dma_direct_free_pages(dev, page, size);  in dma_direct_free_pages()
419 			swiotlb_sync_single_for_device(dev, paddr, sg->length,  in dma_direct_sync_sg_for_device()
423 			arch_sync_dma_for_device(paddr, sg->length,  in dma_direct_sync_sg_for_device()
442 			arch_sync_dma_for_cpu(paddr, sg->length, dir);  in dma_direct_sync_sg_for_cpu()
445 			swiotlb_sync_single_for_cpu(dev, paddr, sg->length,  in dma_direct_sync_sg_for_cpu()
449 			arch_dma_mark_clean(paddr, sg->length);  in dma_direct_sync_sg_for_cpu()
470 			dma_direct_unmap_page(dev, sg->dma_address,  in dma_direct_unmap_sg()
499 				ret = -EREMOTEIO;  in dma_direct_map_sg()
504 		sg->dma_address = dma_direct_map_page(dev, sg_page(sg),  in dma_direct_map_sg()
505 				sg->offset, sg->length, dir, attrs);  in dma_direct_map_sg()
506 		if (sg->dma_address == DMA_MAPPING_ERROR) {  in dma_direct_map_sg()
507 			ret = -EIO;  in dma_direct_map_sg()
510 		sg_dma_len(sg) = sg->length;  in dma_direct_map_sg()
521 		size_t size, enum dma_data_direction dir, unsigned long attrs)  in dma_direct_map_resource()  argument
525 	if (unlikely(!dma_capable(dev, dma_addr, size, false))) {  in dma_direct_map_resource()
528 			     &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);  in dma_direct_map_resource()
537 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_direct_get_sgtable()  argument
540 	struct page *page = dma_direct_to_page(dev, dma_addr);  in dma_direct_get_sgtable()  local
545 		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in dma_direct_get_sgtable()
556 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_direct_mmap()  argument
560 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_direct_mmap()
562 	int ret = -ENXIO;  in dma_direct_mmap()
564 	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);  in dma_direct_mmap()
566 		vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);  in dma_direct_mmap()
568 	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))  in dma_direct_mmap()
570 	if (dma_mmap_from_global_coherent(vma, cpu_addr, size, &ret))  in dma_direct_mmap()
573 	if (vma->vm_pgoff >= count || user_count > count - vma->vm_pgoff)  in dma_direct_mmap()
574 		return -ENXIO;  in dma_direct_mmap()
575 	return remap_pfn_range(vma, vma->vm_start, pfn + vma->vm_pgoff,  in dma_direct_mmap()
576 			user_count << PAGE_SHIFT, vma->vm_page_prot);  in dma_direct_mmap()
581 	u64 min_mask = (max_pfn - 1) << PAGE_SHIFT;  in dma_direct_supported()
584 	 * Because 32-bit DMA masks are so common we expect every architecture  in dma_direct_supported()
585 	 * to be able to satisfy them - either by not supporting more physical  in dma_direct_supported()
604 	/* If SWIOTLB is active, use its maximum mapping size */  in dma_direct_max_mapping_size()
618  * dma_direct_set_offset - Assign scalar offset for a single DMA range.
622  * @size:	size of the region.
625  * be discovered by "dma-ranges".
627  * It returns -ENOMEM if out of memory, -EINVAL if a map
634 			 dma_addr_t dma_start, u64 size)  in dma_direct_set_offset()  argument
637 	u64 offset = (u64)cpu_start - (u64)dma_start;  in dma_direct_set_offset()
639 	if (dev->dma_range_map) {  in dma_direct_set_offset()
641 		return -EINVAL;  in dma_direct_set_offset()
649 		return -ENOMEM;  in dma_direct_set_offset()
653 	map[0].size = size;  in dma_direct_set_offset()
654 	dev->dma_range_map = map;  in dma_direct_set_offset()