3  *  linux/arch/arm/mm/dma-mapping.c
7  *  DMA uncached mapping support.
288 		 * Clear previous low-memory mapping to ensure that the  in dma_contiguous_remap()
609  * Free a buffer as defined by the above mapping.
683  * Use the driver DMA support - see dma-mapping.h (dma_sync_*)
758 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping);
760 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping,  in __alloc_iova()  argument
766 	size_t mapping_size = mapping->bits << PAGE_SHIFT;  in __alloc_iova()
777 	spin_lock_irqsave(&mapping->lock, flags);  in __alloc_iova()
778 	for (i = 0; i < mapping->nr_bitmaps; i++) {  in __alloc_iova()
779 		start = bitmap_find_next_zero_area(mapping->bitmaps[i],  in __alloc_iova()
780 				mapping->bits, 0, count, align);  in __alloc_iova()
782 		if (start > mapping->bits)  in __alloc_iova()
785 		bitmap_set(mapping->bitmaps[i], start, count);  in __alloc_iova()
790 	 * No unused range found. Try to extend the existing mapping  in __alloc_iova()
794 	if (i == mapping->nr_bitmaps) {  in __alloc_iova()
795 		if (extend_iommu_mapping(mapping)) {  in __alloc_iova()
796 			spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
800 		start = bitmap_find_next_zero_area(mapping->bitmaps[i],  in __alloc_iova()
801 				mapping->bits, 0, count, align);  in __alloc_iova()
803 		if (start > mapping->bits) {  in __alloc_iova()
804 			spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
808 		bitmap_set(mapping->bitmaps[i], start, count);  in __alloc_iova()
810 	spin_unlock_irqrestore(&mapping->lock, flags);  in __alloc_iova()
812 	iova = mapping->base + (mapping_size * i);  in __alloc_iova()
818 static inline void __free_iova(struct dma_iommu_mapping *mapping,  in __free_iova()  argument
822 	size_t mapping_size = mapping->bits << PAGE_SHIFT;  in __free_iova()
830 	bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size;  in __free_iova()
831 	BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions);  in __free_iova()
833 	bitmap_base = mapping->base + mapping_size * bitmap_index;  in __free_iova()
848 	spin_lock_irqsave(&mapping->lock, flags);  in __free_iova()
849 	bitmap_clear(mapping->bitmaps[bitmap_index], start, count);  in __free_iova()
850 	spin_unlock_irqrestore(&mapping->lock, flags);  in __free_iova()
966  * Create a mapping in device IO address space for specified pages
972 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __iommu_create_mapping()  local
977 	dma_addr = __alloc_iova(mapping, size);  in __iommu_create_mapping()
994 		ret = iommu_map(mapping->domain, iova, phys, len,  in __iommu_create_mapping()
1003 	iommu_unmap(mapping->domain, dma_addr, iova-dma_addr);  in __iommu_create_mapping()
1004 	__free_iova(mapping, dma_addr, size);  in __iommu_create_mapping()
1010 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __iommu_remove_mapping()  local
1019 	iommu_unmap(mapping->domain, iova, size);  in __iommu_remove_mapping()
1020 	__free_iova(mapping, iova, size);  in __iommu_remove_mapping()
1155  * free a page as defined by the above mapping.
1204 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in __map_sg_chunk()  local
1214 	iova_base = iova = __alloc_iova(mapping, size);  in __map_sg_chunk()
1227 		ret = iommu_map(mapping->domain, iova, phys, len, prot);  in __map_sg_chunk()
1237 	iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE);  in __map_sg_chunk()
1238 	__free_iova(mapping, iova_base, size);  in __map_sg_chunk()
1386 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_map_page()  local
1393 	dma_addr = __alloc_iova(mapping, len);  in arm_iommu_map_page()
1399 	ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot);  in arm_iommu_map_page()
1405 	__free_iova(mapping, dma_addr, len);  in arm_iommu_map_page()
1421 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_unmap_page()  local
1431 		page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_unmap_page()
1435 	iommu_unmap(mapping->domain, iova, len);  in arm_iommu_unmap_page()
1436 	__free_iova(mapping, iova, len);  in arm_iommu_unmap_page()
1450 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_map_resource()  local
1457 	dma_addr = __alloc_iova(mapping, len);  in arm_iommu_map_resource()
1463 	ret = iommu_map(mapping->domain, dma_addr, addr, len, prot);  in arm_iommu_map_resource()
1469 	__free_iova(mapping, dma_addr, len);  in arm_iommu_map_resource()
1484 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_unmap_resource()  local
1492 	iommu_unmap(mapping->domain, iova, len);  in arm_iommu_unmap_resource()
1493 	__free_iova(mapping, iova, len);  in arm_iommu_unmap_resource()
1499 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_sync_single_for_cpu()  local
1507 	page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_sync_single_for_cpu()
1514 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_iommu_sync_single_for_device()  local
1522 	page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova));  in arm_iommu_sync_single_for_device()
1552  * Creates a mapping structure which holds information about used/unused
1554  * mapping with IOMMU aware functions.
1556  * The client device need to be attached to the mapping with
1564 	struct dma_iommu_mapping *mapping;  in arm_iommu_create_mapping()  local
1580 	mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL);  in arm_iommu_create_mapping()
1581 	if (!mapping)  in arm_iommu_create_mapping()
1584 	mapping->bitmap_size = bitmap_size;  in arm_iommu_create_mapping()
1585 	mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *),  in arm_iommu_create_mapping()
1587 	if (!mapping->bitmaps)  in arm_iommu_create_mapping()
1590 	mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL);  in arm_iommu_create_mapping()
1591 	if (!mapping->bitmaps[0])  in arm_iommu_create_mapping()
1594 	mapping->nr_bitmaps = 1;  in arm_iommu_create_mapping()
1595 	mapping->extensions = extensions;  in arm_iommu_create_mapping()
1596 	mapping->base = base;  in arm_iommu_create_mapping()
1597 	mapping->bits = BITS_PER_BYTE * bitmap_size;  in arm_iommu_create_mapping()
1599 	spin_lock_init(&mapping->lock);  in arm_iommu_create_mapping()
1601 	mapping->domain = iommu_domain_alloc(bus);  in arm_iommu_create_mapping()
1602 	if (!mapping->domain)  in arm_iommu_create_mapping()
1605 	kref_init(&mapping->kref);  in arm_iommu_create_mapping()
1606 	return mapping;  in arm_iommu_create_mapping()
1608 	kfree(mapping->bitmaps[0]);  in arm_iommu_create_mapping()
1610 	kfree(mapping->bitmaps);  in arm_iommu_create_mapping()
1612 	kfree(mapping);  in arm_iommu_create_mapping()
1621 	struct dma_iommu_mapping *mapping =  in release_iommu_mapping()  local
1624 	iommu_domain_free(mapping->domain);  in release_iommu_mapping()
1625 	for (i = 0; i < mapping->nr_bitmaps; i++)  in release_iommu_mapping()
1626 		kfree(mapping->bitmaps[i]);  in release_iommu_mapping()
1627 	kfree(mapping->bitmaps);  in release_iommu_mapping()
1628 	kfree(mapping);  in release_iommu_mapping()
1631 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping)  in extend_iommu_mapping()  argument
1635 	if (mapping->nr_bitmaps >= mapping->extensions)  in extend_iommu_mapping()
1638 	next_bitmap = mapping->nr_bitmaps;  in extend_iommu_mapping()
1639 	mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size,  in extend_iommu_mapping()
1641 	if (!mapping->bitmaps[next_bitmap])  in extend_iommu_mapping()
1644 	mapping->nr_bitmaps++;  in extend_iommu_mapping()
1649 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping)  in arm_iommu_release_mapping()  argument
1651 	if (mapping)  in arm_iommu_release_mapping()
1652 		kref_put(&mapping->kref, release_iommu_mapping);  in arm_iommu_release_mapping()
1657 				     struct dma_iommu_mapping *mapping)  in __arm_iommu_attach_device()  argument
1661 	err = iommu_attach_device(mapping->domain, dev);  in __arm_iommu_attach_device()
1665 	kref_get(&mapping->kref);  in __arm_iommu_attach_device()
1666 	to_dma_iommu_mapping(dev) = mapping;  in __arm_iommu_attach_device()
1675  * @mapping: io address space mapping structure (returned from
1678  * Attaches specified io address space mapping to the provided device.
1683  * mapping.
1686 			    struct dma_iommu_mapping *mapping)  in arm_iommu_attach_device()  argument
1690 	err = __arm_iommu_attach_device(dev, mapping);  in arm_iommu_attach_device()
1708 	struct dma_iommu_mapping *mapping;  in arm_iommu_detach_device()  local
1710 	mapping = to_dma_iommu_mapping(dev);  in arm_iommu_detach_device()
1711 	if (!mapping) {  in arm_iommu_detach_device()
1716 	iommu_detach_device(mapping->domain, dev);  in arm_iommu_detach_device()
1717 	kref_put(&mapping->kref, release_iommu_mapping);  in arm_iommu_detach_device()
1728 	struct dma_iommu_mapping *mapping;  in arm_setup_iommu_dma_ops()  local
1730 	mapping = arm_iommu_create_mapping(dev->bus, dma_base, size);  in arm_setup_iommu_dma_ops()
1731 	if (IS_ERR(mapping)) {  in arm_setup_iommu_dma_ops()
1732 		pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n",  in arm_setup_iommu_dma_ops()
1737 	if (__arm_iommu_attach_device(dev, mapping)) {  in arm_setup_iommu_dma_ops()
1740 		arm_iommu_release_mapping(mapping);  in arm_setup_iommu_dma_ops()
1749 	struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev);  in arm_teardown_iommu_dma_ops()  local
1751 	if (!mapping)  in arm_teardown_iommu_dma_ops()
1755 	arm_iommu_release_mapping(mapping);  in arm_teardown_iommu_dma_ops()