1 // SPDX-License-Identifier: GPL-2.0-only
7  * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
10  * and on exynos-iommu.c
18 #include <linux/dma-iommu.h>
19 #include <linux/dma-mapping.h>
22 #include <linux/io-pgtable.h>
143  * @pgtbl_ops: pagetable ops allocated by io-pgtable
180 	for (i = 0, stream_map = &(base)->stream_maps[0];                      \
181 	     i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
182 	     stream_map = &(base)->stream_maps[++i])
197 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_enable_translation()
199 		       stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_enable_translation()
206 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_disable_dma()
207 		writel(0, stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_disable_dma()
215 	WARN_ON(!stream_map->dart->supports_bypass);  in apple_dart_hw_enable_bypass()
216 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_enable_bypass()
218 		       stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_enable_bypass()
226 	WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1));  in apple_dart_hw_set_ttbr()
227 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_set_ttbr()
229 		       stream_map->dart->regs + DART_TTBR(sid, idx));  in apple_dart_hw_set_ttbr()
237 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_clear_ttbr()
238 		writel(0, stream_map->dart->regs + DART_TTBR(sid, idx));  in apple_dart_hw_clear_ttbr()
258 	spin_lock_irqsave(&stream_map->dart->lock, flags);  in apple_dart_hw_stream_command()
260 	writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT);  in apple_dart_hw_stream_command()
261 	writel(command, stream_map->dart->regs + DART_STREAM_COMMAND);  in apple_dart_hw_stream_command()
264 		stream_map->dart->regs + DART_STREAM_COMMAND, command_reg,  in apple_dart_hw_stream_command()
268 	spin_unlock_irqrestore(&stream_map->dart->lock, flags);  in apple_dart_hw_stream_command()
271 		dev_err(stream_map->dart->dev,  in apple_dart_hw_stream_command()
273 			command, stream_map->sidmap);  in apple_dart_hw_stream_command()
292 	config = readl(dart->regs + DART_CONFIG);  in apple_dart_hw_reset()
294 		dev_err(dart->dev, "DART is locked down until reboot: %08x\n",  in apple_dart_hw_reset()
296 		return -EINVAL;  in apple_dart_hw_reset()
305 	writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);  in apple_dart_hw_reset()
317 		stream_map.dart = domain_stream_map->dart;  in apple_dart_domain_flush_tlb()
318 		stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap);  in apple_dart_domain_flush_tlb()
344 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_iova_to_phys()
349 	return ops->iova_to_phys(ops, iova);  in apple_dart_iova_to_phys()
358 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_map_pages()
361 		return -ENODEV;  in apple_dart_map_pages()
363 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,  in apple_dart_map_pages()
373 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_unmap_pages()
375 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);  in apple_dart_unmap_pages()
384 		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;  in apple_dart_setup_translation()
386 	for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)  in apple_dart_setup_translation()
388 				       pgtbl_cfg->apple_dart_cfg.ttbr[i]);  in apple_dart_setup_translation()
400 	struct apple_dart *dart = cfg->stream_maps[0].dart;  in apple_dart_finalize_domain()
405 	mutex_lock(&dart_domain->init_lock);  in apple_dart_finalize_domain()
407 	if (dart_domain->finalized)  in apple_dart_finalize_domain()
411 		dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;  in apple_dart_finalize_domain()
412 		atomic64_set(&dart_domain->stream_maps[i].sidmap,  in apple_dart_finalize_domain()
413 			     cfg->stream_maps[i].sidmap);  in apple_dart_finalize_domain()
417 		.pgsize_bitmap = dart->pgsize,  in apple_dart_finalize_domain()
421 		.iommu_dev = dart->dev,  in apple_dart_finalize_domain()
424 	dart_domain->pgtbl_ops =  in apple_dart_finalize_domain()
426 	if (!dart_domain->pgtbl_ops) {  in apple_dart_finalize_domain()
427 		ret = -ENOMEM;  in apple_dart_finalize_domain()
431 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;  in apple_dart_finalize_domain()
432 	domain->geometry.aperture_start = 0;  in apple_dart_finalize_domain()
433 	domain->geometry.aperture_end = DMA_BIT_MASK(32);  in apple_dart_finalize_domain()
434 	domain->geometry.force_aperture = true;  in apple_dart_finalize_domain()
436 	dart_domain->finalized = true;  in apple_dart_finalize_domain()
439 	mutex_unlock(&dart_domain->init_lock);  in apple_dart_finalize_domain()
452 			return -EINVAL;  in apple_dart_mod_streams()
472 	return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,  in apple_dart_domain_add_streams()
479 	return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,  in apple_dart_domain_remove_streams()
491 	if (cfg->stream_maps[0].dart->force_bypass &&  in apple_dart_attach_dev()
492 	    domain->type != IOMMU_DOMAIN_IDENTITY)  in apple_dart_attach_dev()
493 		return -EINVAL;  in apple_dart_attach_dev()
494 	if (!cfg->stream_maps[0].dart->supports_bypass &&  in apple_dart_attach_dev()
495 	    domain->type == IOMMU_DOMAIN_IDENTITY)  in apple_dart_attach_dev()
496 		return -EINVAL;  in apple_dart_attach_dev()
502 	switch (domain->type) {  in apple_dart_attach_dev()
536 	if (domain->type == IOMMU_DOMAIN_DMA ||  in apple_dart_detach_dev()
537 	    domain->type == IOMMU_DOMAIN_UNMANAGED)  in apple_dart_detach_dev()
548 		return ERR_PTR(-ENODEV);  in apple_dart_probe_device()
552 			dev, stream_map->dart->dev,  in apple_dart_probe_device()
555 	return &cfg->stream_maps[0].dart->iommu;  in apple_dart_probe_device()
581 	iommu_get_dma_cookie(&dart_domain->domain);  in apple_dart_domain_alloc()
582 	mutex_init(&dart_domain->init_lock);  in apple_dart_domain_alloc()
586 		dart_domain->finalized = true;  in apple_dart_domain_alloc()
588 	return &dart_domain->domain;  in apple_dart_domain_alloc()
595 	if (dart_domain->pgtbl_ops)  in apple_dart_domain_free()
596 		free_io_pgtable_ops(dart_domain->pgtbl_ops);  in apple_dart_domain_free()
604 	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);  in apple_dart_of_xlate()
609 	if (args->args_count != 1)  in apple_dart_of_xlate()
610 		return -EINVAL;  in apple_dart_of_xlate()
611 	sid = args->args[0];  in apple_dart_of_xlate()
616 		return -ENOMEM;  in apple_dart_of_xlate()
619 	cfg_dart = cfg->stream_maps[0].dart;  in apple_dart_of_xlate()
621 		if (cfg_dart->supports_bypass != dart->supports_bypass)  in apple_dart_of_xlate()
622 			return -EINVAL;  in apple_dart_of_xlate()
623 		if (cfg_dart->force_bypass != dart->force_bypass)  in apple_dart_of_xlate()
624 			return -EINVAL;  in apple_dart_of_xlate()
625 		if (cfg_dart->pgsize != dart->pgsize)  in apple_dart_of_xlate()
626 			return -EINVAL;  in apple_dart_of_xlate()
630 		if (cfg->stream_maps[i].dart == dart) {  in apple_dart_of_xlate()
631 			cfg->stream_maps[i].sidmap |= 1 << sid;  in apple_dart_of_xlate()
636 		if (!cfg->stream_maps[i].dart) {  in apple_dart_of_xlate()
637 			cfg->stream_maps[i].dart = dart;  in apple_dart_of_xlate()
638 			cfg->stream_maps[i].sidmap = 1 << sid;  in apple_dart_of_xlate()
643 	return -EINVAL;  in apple_dart_of_xlate()
657 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_release_group()
658 			stream_map->dart->sid2group[sid] = NULL;  in apple_dart_release_group()
671 	struct iommu_group *res = ERR_PTR(-EINVAL);  in apple_dart_device_group()
676 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) {  in apple_dart_device_group()
678 				stream_map->dart->sid2group[sid];  in apple_dart_device_group()
681 				res = ERR_PTR(-EINVAL);  in apple_dart_device_group()
701 	res = ERR_PTR(-ENOMEM);  in apple_dart_device_group()
716 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_device_group()
717 			stream_map->dart->sid2group[sid] = group;  in apple_dart_device_group()
730 	if (cfg->stream_maps[0].dart->force_bypass)  in apple_dart_def_domain_type()
732 	if (!cfg->stream_maps[0].dart->supports_bypass)  in apple_dart_def_domain_type()
754 	.pgsize_bitmap = -1UL, /* Restricted during dart probe */
761 	u32 error = readl(dart->regs + DART_ERROR);  in apple_dart_irq()
763 	u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO);  in apple_dart_irq()
764 	u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI);  in apple_dart_irq()
786 		dart->dev,  in apple_dart_irq()
790 	writel(error, dart->regs + DART_ERROR);  in apple_dart_irq()
821 	struct device *dev = &pdev->dev;  in apple_dart_probe()
825 		return -ENOMEM;  in apple_dart_probe()
827 	dart->dev = dev;  in apple_dart_probe()
828 	spin_lock_init(&dart->lock);  in apple_dart_probe()
833 		return -EINVAL;  in apple_dart_probe()
836 	dart->regs = devm_ioremap_resource(dev, res);  in apple_dart_probe()
837 	if (IS_ERR(dart->regs))  in apple_dart_probe()
838 		return PTR_ERR(dart->regs);  in apple_dart_probe()
840 	dart->irq = platform_get_irq(pdev, 0);  in apple_dart_probe()
841 	if (dart->irq < 0)  in apple_dart_probe()
842 		return -ENODEV;  in apple_dart_probe()
844 	ret = devm_clk_bulk_get_all(dev, &dart->clks);  in apple_dart_probe()
847 	dart->num_clks = ret;  in apple_dart_probe()
849 	ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);  in apple_dart_probe()
857 	dart_params[0] = readl(dart->regs + DART_PARAMS1);  in apple_dart_probe()
858 	dart_params[1] = readl(dart->regs + DART_PARAMS2);  in apple_dart_probe()
859 	dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]);  in apple_dart_probe()
860 	dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT;  in apple_dart_probe()
861 	dart->force_bypass = dart->pgsize > PAGE_SIZE;  in apple_dart_probe()
863 	ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED,  in apple_dart_probe()
864 			  "apple-dart fault handler", dart);  in apple_dart_probe()
874 	ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",  in apple_dart_probe()
875 				     dev_name(&pdev->dev));  in apple_dart_probe()
879 	ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);  in apple_dart_probe()
884 		&pdev->dev,  in apple_dart_probe()
886 		dart->pgsize, dart->supports_bypass, dart->force_bypass);  in apple_dart_probe()
890 	iommu_device_sysfs_remove(&dart->iommu);  in apple_dart_probe()
894 	free_irq(dart->irq, dart);  in apple_dart_probe()
896 	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);  in apple_dart_probe()
906 	free_irq(dart->irq, dart);  in apple_dart_remove()
909 	iommu_device_unregister(&dart->iommu);  in apple_dart_remove()
910 	iommu_device_sysfs_remove(&dart->iommu);  in apple_dart_remove()
912 	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);  in apple_dart_remove()
918 	{ .compatible = "apple,t8103-dart", .data = NULL },
925 		.name			= "apple-dart",