1 // SPDX-License-Identifier: GPL-2.0-only
5  * Use the integrated AGP GART in the Hammer northbridge as an IOMMU for PCI.
9  * See Documentation/core-api/dma-api-howto.rst for the interface specification.
29 #include <linux/iommu-helper.h>
34 #include <linux/dma-direct.h>
35 #include <linux/dma-map-ops.h>
38 #include <asm/iommu.h>
54  * If this is disabled the IOMMU will use an optimized flushing strategy
105 	if (offset == -1) {  in alloc_iommu()
111 	if (offset != -1) {  in alloc_iommu()
152 /* Debugging aid for drivers that don't free their IOMMU tables */
169 	 * Ran out of IOMMU space for this operation. This is very bad.  in iommu_full()
178 	dev_err(dev, "PCI-DMA: Out of IOMMU space for %lu bytes\n", size);  in iommu_full()
196 /* Map a single continuous physical area into the IOMMU.
197  * Caller needs to check if the iommu is needed and flush.
210 	if (iommu_page == -1) {  in dma_map_area()
226 /* Map a single area into the IOMMU */
267 	iommu_page = (dma_addr - iommu_bus_base)>>PAGE_SHIFT;  in gart_unmap_page()
285 		if (!s->dma_length || !s->length)  in gart_unmap_sg()
287 		gart_unmap_page(dev, s->dma_address, s->dma_length, dir, 0);  in gart_unmap_sg()
305 		if (nonforced_iommu(dev, addr, s->length)) {  in dma_map_sg_nonforce()
306 			addr = dma_map_area(dev, addr, s->length, dir, 0);  in dma_map_sg_nonforce()
315 		s->dma_address = addr;  in dma_map_sg_nonforce()
316 		s->dma_length = s->length;  in dma_map_sg_nonforce()
323 /* Map multiple scatterlist entries continuous into the first. */
333 	if (iommu_start == -1)  in __dma_map_cont()
334 		return -1;  in __dma_map_cont()
338 		unsigned long phys_addr = s->dma_address;  in __dma_map_cont()
340 		BUG_ON(s != start && s->offset);  in __dma_map_cont()
342 			sout->dma_address = iommu_bus_base;  in __dma_map_cont()
343 			sout->dma_address += iommu_page*PAGE_SIZE + s->offset;  in __dma_map_cont()
344 			sout->dma_length = s->length;  in __dma_map_cont()
346 			sout->dma_length += s->length;  in __dma_map_cont()
350 		pages = iommu_num_pages(s->offset, s->length, PAGE_SIZE);  in __dma_map_cont()
351 		while (pages--) {  in __dma_map_cont()
357 	BUG_ON(iommu_page - iommu_start != pages);  in __dma_map_cont()
368 		sout->dma_address = start->dma_address;  in dma_map_cont()
369 		sout->dma_length = start->length;  in dma_map_cont()
376  * DMA map all entries in a scatterlist.
402 		s->dma_address = addr;  in gart_map_sg()
403 		BUG_ON(s->length == 0);  in gart_map_sg()
405 		nextneed = need_iommu(dev, addr, s->length);  in gart_map_sg()
414 			if (!iommu_merge || !nextneed || !need || s->offset ||  in gart_map_sg()
415 			    (s->length + seg_size > max_seg_size) ||  in gart_map_sg()
416 			    (ps->offset + ps->length) % PAGE_SIZE) {  in gart_map_sg()
417 				if (dma_map_cont(dev, start_sg, i - start,  in gart_map_sg()
430 		seg_size += s->length;  in gart_map_sg()
432 		pages += iommu_num_pages(s->offset, s->length, PAGE_SIZE);  in gart_map_sg()
435 	if (dma_map_cont(dev, start_sg, i - start, sgmap, pages, need) < 0)  in gart_map_sg()
441 		sgmap->dma_length = 0;  in gart_map_sg()
460 		s->dma_address = DMA_MAPPING_ERROR;  in gart_map_sg()
464 /* allocate and map a coherent mapping */
473 	    !force_iommu || dev->coherent_dma_mask <= DMA_BIT_MASK(24))  in gart_alloc_coherent()
477 			DMA_BIDIRECTIONAL, (1UL << get_order(size)) - 1);  in gart_alloc_coherent()
509 	iommu_size -= round_up(a, PMD_PAGE_SIZE) - a;  in check_iommu_size()
512 		pr_warn("PCI-DMA: Warning: Small IOMMU %luMB."  in check_iommu_size()
548 		struct pci_dev *dev = node_to_amd_nb(i)->misc;  in enable_gart_translations()
553 	/* Flush the GART-TLB to remove stale entries */  in enable_gart_translations()
582 	pr_info("PCI-DMA: Restoring GART aperture settings\n");  in gart_fixup_northbridges()
585 		struct pci_dev *dev = node_to_amd_nb(i)->misc;  in gart_fixup_northbridges()
589 		 * step.  Restore the pre-suspend aperture settings.  in gart_fixup_northbridges()
598 	pr_info("PCI-DMA: Resuming GART IOMMU\n");  in gart_resume()
622 	pr_info("PCI-DMA: Disabling AGP.\n");  in init_amd_gatt()
624 	aper_size = aper_base = info->aper_size = 0;  in init_amd_gatt()
627 		dev = node_to_amd_nb(i)->misc;  in init_amd_gatt()
642 	info->aper_base = aper_base;  in init_amd_gatt()
643 	info->aper_size = aper_size >> 20;  in init_amd_gatt()
659 	pr_info("PCI-DMA: aperture base @ %x size %u KB\n",  in init_amd_gatt()
666 	pr_warn("PCI-DMA: More than 4GB of RAM and no IOMMU - falling back to iommu=soft.\n");  in init_amd_gatt()
667 	return -1;  in init_amd_gatt()
700 		dev = node_to_amd_nb(i)->misc;  in gart_iommu_shutdown()
735 			pr_warn("More than 4GB of memory but GART IOMMU not available.\n");  in gart_iommu_init()
736 			pr_warn("falling back to iommu=soft.\n");  in gart_iommu_init()
741 	/* need to map that range */  in gart_iommu_init()
751 	pr_info("PCI-DMA: using GART IOMMU.\n");  in gart_iommu_init()
758 		panic("Cannot allocate iommu bitmap\n");  in gart_iommu_init()
760 	pr_info("PCI-DMA: Reserving %luMB of IOMMU area in the AGP aperture\n",  in gart_iommu_init()
764 	iommu_start		= aper_size - iommu_size;  in gart_iommu_init()
769 	 * Unmap the IOMMU part of the GART. The alias of the page is  in gart_iommu_init()
784 	 * do an explicit, full-scale wbinvd() _after_ having marked all  in gart_iommu_init()
785 	 * the pages as Not-Present:  in gart_iommu_init()
805 		panic("Cannot allocate iommu scratch page");  in gart_iommu_init()
830 	/* duplicated from pci-dma.c */  in gart_parse_options()