1 // SPDX-License-Identifier: GPL-2.0-only
7  * Based on arm/arm-smmu/arm-ssmu.c and arm/arm-smmu-v3/arm-smmu-v3.c
10  * and on exynos-iommu.c
18 #include <linux/dma-mapping.h>
21 #include <linux/io-pgtable.h>
35 #include "dma-iommu.h"
94  * @hw: SoC-specific hardware data
153  * @pgtbl_ops: pagetable ops allocated by io-pgtable
190 	for (i = 0, stream_map = &(base)->stream_maps[0];                      \
191 	     i < MAX_DARTS_PER_DEVICE && stream_map->dart;                     \
192 	     stream_map = &(base)->stream_maps[++i])
207 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_enable_translation()
209 		       stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_enable_translation()
216 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_disable_dma()
217 		writel(0, stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_disable_dma()
225 	WARN_ON(!stream_map->dart->supports_bypass);  in apple_dart_hw_enable_bypass()
226 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_enable_bypass()
228 		       stream_map->dart->regs + DART_TCR(sid));  in apple_dart_hw_enable_bypass()
236 	WARN_ON(paddr & ((1 << DART_TTBR_SHIFT) - 1));  in apple_dart_hw_set_ttbr()
237 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_set_ttbr()
239 		       stream_map->dart->regs + DART_TTBR(sid, idx));  in apple_dart_hw_set_ttbr()
247 	for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_hw_clear_ttbr()
248 		writel(0, stream_map->dart->regs + DART_TTBR(sid, idx));  in apple_dart_hw_clear_ttbr()
268 	spin_lock_irqsave(&stream_map->dart->lock, flags);  in apple_dart_hw_stream_command()
270 	writel(stream_map->sidmap, stream_map->dart->regs + DART_STREAM_SELECT);  in apple_dart_hw_stream_command()
271 	writel(command, stream_map->dart->regs + DART_STREAM_COMMAND);  in apple_dart_hw_stream_command()
274 		stream_map->dart->regs + DART_STREAM_COMMAND, command_reg,  in apple_dart_hw_stream_command()
278 	spin_unlock_irqrestore(&stream_map->dart->lock, flags);  in apple_dart_hw_stream_command()
281 		dev_err(stream_map->dart->dev,  in apple_dart_hw_stream_command()
283 			command, stream_map->sidmap);  in apple_dart_hw_stream_command()
302 	config = readl(dart->regs + DART_CONFIG);  in apple_dart_hw_reset()
304 		dev_err(dart->dev, "DART is locked down until reboot: %08x\n",  in apple_dart_hw_reset()
306 		return -EINVAL;  in apple_dart_hw_reset()
315 	writel(DART_STREAM_ALL, dart->regs + DART_STREAMS_ENABLE);  in apple_dart_hw_reset()
318 	writel(readl(dart->regs + DART_ERROR), dart->regs + DART_ERROR);  in apple_dart_hw_reset()
330 		stream_map.dart = domain_stream_map->dart;  in apple_dart_domain_flush_tlb()
331 		stream_map.sidmap = atomic64_read(&domain_stream_map->sidmap);  in apple_dart_domain_flush_tlb()
357 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_iova_to_phys()
362 	return ops->iova_to_phys(ops, iova);  in apple_dart_iova_to_phys()
371 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_map_pages()
374 		return -ENODEV;  in apple_dart_map_pages()
376 	return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp,  in apple_dart_map_pages()
386 	struct io_pgtable_ops *ops = dart_domain->pgtbl_ops;  in apple_dart_unmap_pages()
388 	return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);  in apple_dart_unmap_pages()
397 		&io_pgtable_ops_to_pgtable(domain->pgtbl_ops)->cfg;  in apple_dart_setup_translation()
399 	for (i = 0; i < pgtbl_cfg->apple_dart_cfg.n_ttbrs; ++i)  in apple_dart_setup_translation()
401 				       pgtbl_cfg->apple_dart_cfg.ttbr[i]);  in apple_dart_setup_translation()
413 	struct apple_dart *dart = cfg->stream_maps[0].dart;  in apple_dart_finalize_domain()
418 	mutex_lock(&dart_domain->init_lock);  in apple_dart_finalize_domain()
420 	if (dart_domain->finalized)  in apple_dart_finalize_domain()
424 		dart_domain->stream_maps[i].dart = cfg->stream_maps[i].dart;  in apple_dart_finalize_domain()
425 		atomic64_set(&dart_domain->stream_maps[i].sidmap,  in apple_dart_finalize_domain()
426 			     cfg->stream_maps[i].sidmap);  in apple_dart_finalize_domain()
430 		.pgsize_bitmap = dart->pgsize,  in apple_dart_finalize_domain()
432 		.oas = dart->hw->oas,  in apple_dart_finalize_domain()
434 		.iommu_dev = dart->dev,  in apple_dart_finalize_domain()
437 	dart_domain->pgtbl_ops =  in apple_dart_finalize_domain()
438 		alloc_io_pgtable_ops(dart->hw->fmt, &pgtbl_cfg, domain);  in apple_dart_finalize_domain()
439 	if (!dart_domain->pgtbl_ops) {  in apple_dart_finalize_domain()
440 		ret = -ENOMEM;  in apple_dart_finalize_domain()
444 	domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;  in apple_dart_finalize_domain()
445 	domain->geometry.aperture_start = 0;  in apple_dart_finalize_domain()
446 	domain->geometry.aperture_end = DMA_BIT_MASK(32);  in apple_dart_finalize_domain()
447 	domain->geometry.force_aperture = true;  in apple_dart_finalize_domain()
449 	dart_domain->finalized = true;  in apple_dart_finalize_domain()
452 	mutex_unlock(&dart_domain->init_lock);  in apple_dart_finalize_domain()
465 			return -EINVAL;  in apple_dart_mod_streams()
485 	return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,  in apple_dart_domain_add_streams()
492 	return apple_dart_mod_streams(domain->stream_maps, cfg->stream_maps,  in apple_dart_domain_remove_streams()
504 	if (cfg->stream_maps[0].dart->force_bypass &&  in apple_dart_attach_dev()
505 	    domain->type != IOMMU_DOMAIN_IDENTITY)  in apple_dart_attach_dev()
506 		return -EINVAL;  in apple_dart_attach_dev()
507 	if (!cfg->stream_maps[0].dart->supports_bypass &&  in apple_dart_attach_dev()
508 	    domain->type == IOMMU_DOMAIN_IDENTITY)  in apple_dart_attach_dev()
509 		return -EINVAL;  in apple_dart_attach_dev()
515 	switch (domain->type) {  in apple_dart_attach_dev()
549 	if (domain->type == IOMMU_DOMAIN_DMA ||  in apple_dart_detach_dev()
550 	    domain->type == IOMMU_DOMAIN_UNMANAGED)  in apple_dart_detach_dev()
561 		return ERR_PTR(-ENODEV);  in apple_dart_probe_device()
565 			dev, stream_map->dart->dev,  in apple_dart_probe_device()
568 	return &cfg->stream_maps[0].dart->iommu;  in apple_dart_probe_device()
591 	mutex_init(&dart_domain->init_lock);  in apple_dart_domain_alloc()
595 		dart_domain->finalized = true;  in apple_dart_domain_alloc()
597 	return &dart_domain->domain;  in apple_dart_domain_alloc()
604 	if (dart_domain->pgtbl_ops)  in apple_dart_domain_free()
605 		free_io_pgtable_ops(dart_domain->pgtbl_ops);  in apple_dart_domain_free()
613 	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);  in apple_dart_of_xlate()
618 	if (args->args_count != 1)  in apple_dart_of_xlate()
619 		return -EINVAL;  in apple_dart_of_xlate()
620 	sid = args->args[0];  in apple_dart_of_xlate()
625 		return -ENOMEM;  in apple_dart_of_xlate()
628 	cfg_dart = cfg->stream_maps[0].dart;  in apple_dart_of_xlate()
630 		if (cfg_dart->supports_bypass != dart->supports_bypass)  in apple_dart_of_xlate()
631 			return -EINVAL;  in apple_dart_of_xlate()
632 		if (cfg_dart->force_bypass != dart->force_bypass)  in apple_dart_of_xlate()
633 			return -EINVAL;  in apple_dart_of_xlate()
634 		if (cfg_dart->pgsize != dart->pgsize)  in apple_dart_of_xlate()
635 			return -EINVAL;  in apple_dart_of_xlate()
639 		if (cfg->stream_maps[i].dart == dart) {  in apple_dart_of_xlate()
640 			cfg->stream_maps[i].sidmap |= 1 << sid;  in apple_dart_of_xlate()
645 		if (!cfg->stream_maps[i].dart) {  in apple_dart_of_xlate()
646 			cfg->stream_maps[i].dart = dart;  in apple_dart_of_xlate()
647 			cfg->stream_maps[i].sidmap = 1 << sid;  in apple_dart_of_xlate()
652 	return -EINVAL;  in apple_dart_of_xlate()
666 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_release_group()
667 			stream_map->dart->sid2group[sid] = NULL;  in apple_dart_release_group()
680 	struct iommu_group *res = ERR_PTR(-EINVAL);  in apple_dart_device_group()
685 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS) {  in apple_dart_device_group()
687 				stream_map->dart->sid2group[sid];  in apple_dart_device_group()
690 				res = ERR_PTR(-EINVAL);  in apple_dart_device_group()
710 	res = ERR_PTR(-ENOMEM);  in apple_dart_device_group()
724 		for_each_set_bit(sid, &stream_map->sidmap, DART_MAX_STREAMS)  in apple_dart_device_group()
725 			stream_map->dart->sid2group[sid] = group;  in apple_dart_device_group()
738 	if (cfg->stream_maps[0].dart->force_bypass)  in apple_dart_def_domain_type()
740 	if (!cfg->stream_maps[0].dart->supports_bypass)  in apple_dart_def_domain_type()
765 		list_add_tail(&region->list, head);  in apple_dart_get_resv_regions()
779 	.pgsize_bitmap = -1UL, /* Restricted during dart probe */
798 	u32 error = readl(dart->regs + DART_ERROR);  in apple_dart_irq()
800 	u32 addr_lo = readl(dart->regs + DART_ERROR_ADDR_LO);  in apple_dart_irq()
801 	u32 addr_hi = readl(dart->regs + DART_ERROR_ADDR_HI);  in apple_dart_irq()
823 		dart->dev,  in apple_dart_irq()
827 	writel(error, dart->regs + DART_ERROR);  in apple_dart_irq()
837 	struct device *dev = &pdev->dev;  in apple_dart_probe()
841 		return -ENOMEM;  in apple_dart_probe()
843 	dart->dev = dev;  in apple_dart_probe()
844 	dart->hw = of_device_get_match_data(dev);  in apple_dart_probe()
845 	spin_lock_init(&dart->lock);  in apple_dart_probe()
847 	dart->regs = devm_platform_get_and_ioremap_resource(pdev, 0, &res);  in apple_dart_probe()
848 	if (IS_ERR(dart->regs))  in apple_dart_probe()
849 		return PTR_ERR(dart->regs);  in apple_dart_probe()
853 		return -EINVAL;  in apple_dart_probe()
856 	dart->irq = platform_get_irq(pdev, 0);  in apple_dart_probe()
857 	if (dart->irq < 0)  in apple_dart_probe()
858 		return -ENODEV;  in apple_dart_probe()
860 	ret = devm_clk_bulk_get_all(dev, &dart->clks);  in apple_dart_probe()
863 	dart->num_clks = ret;  in apple_dart_probe()
865 	ret = clk_bulk_prepare_enable(dart->num_clks, dart->clks);  in apple_dart_probe()
873 	dart_params[0] = readl(dart->regs + DART_PARAMS1);  in apple_dart_probe()
874 	dart_params[1] = readl(dart->regs + DART_PARAMS2);  in apple_dart_probe()
875 	dart->pgsize = 1 << FIELD_GET(DART_PARAMS_PAGE_SHIFT, dart_params[0]);  in apple_dart_probe()
876 	dart->supports_bypass = dart_params[1] & DART_PARAMS_BYPASS_SUPPORT;  in apple_dart_probe()
877 	dart->force_bypass = dart->pgsize > PAGE_SIZE;  in apple_dart_probe()
879 	ret = request_irq(dart->irq, apple_dart_irq, IRQF_SHARED,  in apple_dart_probe()
880 			  "apple-dart fault handler", dart);  in apple_dart_probe()
886 	ret = iommu_device_sysfs_add(&dart->iommu, dev, NULL, "apple-dart.%s",  in apple_dart_probe()
887 				     dev_name(&pdev->dev));  in apple_dart_probe()
891 	ret = iommu_device_register(&dart->iommu, &apple_dart_iommu_ops, dev);  in apple_dart_probe()
896 		&pdev->dev,  in apple_dart_probe()
898 		dart->pgsize, dart->supports_bypass, dart->force_bypass);  in apple_dart_probe()
902 	iommu_device_sysfs_remove(&dart->iommu);  in apple_dart_probe()
904 	free_irq(dart->irq, dart);  in apple_dart_probe()
906 	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);  in apple_dart_probe()
916 	free_irq(dart->irq, dart);  in apple_dart_remove()
918 	iommu_device_unregister(&dart->iommu);  in apple_dart_remove()
919 	iommu_device_sysfs_remove(&dart->iommu);  in apple_dart_remove()
921 	clk_bulk_disable_unprepare(dart->num_clks, dart->clks);  in apple_dart_remove()
936 	{ .compatible = "apple,t8103-dart", .data = &apple_dart_hw_t8103 },
937 	{ .compatible = "apple,t6000-dart", .data = &apple_dart_hw_t6000 },
944 		.name			= "apple-dart",