1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (c) 2015-2016 MediaTek Inc.
14 #include <linux/dma-mapping.h>
29 #include <asm/dma-iommu.h>
30 #include <dt-bindings/memory/mtk-memory-port.h>
31 #include <dt-bindings/memory/mt2701-larb-port.h>
76 #define MT2701_M4U_TF_LARB(TF)			(6 - (((TF) >> 13) & 0x7))
123 	return component_bind_all(dev, &data->larb_imu);  in mtk_iommu_v1_bind()
130 	component_unbind_all(dev, &data->larb_imu);  in mtk_iommu_v1_unbind()
147 	for (i = ARRAY_SIZE(mt2701_m4u_in_larb) - 1; i >= 0; i--)  in mt2701_m4u_to_larb()
156 	int larb = mt2701_m4u_to_larb(id);  in mt2701_m4u_to_port()  local
158 	return id - mt2701_m4u_in_larb[larb];  in mt2701_m4u_to_port()
164 			data->base + REG_MMU_INV_SEL);  in mtk_iommu_v1_tlb_flush_all()
165 	writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE);  in mtk_iommu_v1_tlb_flush_all()
176 		data->base + REG_MMU_INV_SEL);  in mtk_iommu_v1_tlb_flush_range()
178 		data->base + REG_MMU_INVLD_START_A);  in mtk_iommu_v1_tlb_flush_range()
179 	writel_relaxed((iova + size - 1) & F_MMU_FAULT_VA_MSK,  in mtk_iommu_v1_tlb_flush_range()
180 		data->base + REG_MMU_INVLD_END_A);  in mtk_iommu_v1_tlb_flush_range()
181 	writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE);  in mtk_iommu_v1_tlb_flush_range()
183 	ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE,  in mtk_iommu_v1_tlb_flush_range()
186 		dev_warn(data->dev,  in mtk_iommu_v1_tlb_flush_range()
191 	writel_relaxed(0, data->base + REG_MMU_CPE_DONE);  in mtk_iommu_v1_tlb_flush_range()
197 	struct mtk_iommu_v1_domain *dom = data->m4u_dom;  in mtk_iommu_v1_isr()
202 	int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST);  in mtk_iommu_v1_isr()
203 	fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA);  in mtk_iommu_v1_isr()
206 	fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA);  in mtk_iommu_v1_isr()
207 	regval = readl_relaxed(data->base + REG_MMU_INT_ID);  in mtk_iommu_v1_isr()
215 	if (report_iommu_fault(&dom->domain, data->dev, fault_iova,  in mtk_iommu_v1_isr()
217 		dev_err_ratelimited(data->dev,  in mtk_iommu_v1_isr()
218 			"fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d\n",  in mtk_iommu_v1_isr()
223 	regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_isr()
225 	writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_isr()
240 	for (i = 0; i < fwspec->num_ids; ++i) {  in mtk_iommu_v1_config()
241 		larbid = mt2701_m4u_to_larb(fwspec->ids[i]);  in mtk_iommu_v1_config()
242 		portid = mt2701_m4u_to_port(fwspec->ids[i]);  in mtk_iommu_v1_config()
243 		larb_mmu = &data->larb_imu[larbid];  in mtk_iommu_v1_config()
245 		dev_dbg(dev, "%s iommu port: %d\n",  in mtk_iommu_v1_config()
249 			larb_mmu->mmu |= MTK_SMI_MMU_EN(portid);  in mtk_iommu_v1_config()
251 			larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid);  in mtk_iommu_v1_config()
257 	struct mtk_iommu_v1_domain *dom = data->m4u_dom;  in mtk_iommu_v1_domain_finalise()
259 	spin_lock_init(&dom->pgtlock);  in mtk_iommu_v1_domain_finalise()
261 	dom->pgt_va = dma_alloc_coherent(data->dev, M2701_IOMMU_PGT_SIZE,  in mtk_iommu_v1_domain_finalise()
262 					 &dom->pgt_pa, GFP_KERNEL);  in mtk_iommu_v1_domain_finalise()
263 	if (!dom->pgt_va)  in mtk_iommu_v1_domain_finalise()
264 		return -ENOMEM;  in mtk_iommu_v1_domain_finalise()
266 	writel(dom->pgt_pa, data->base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_domain_finalise()
268 	dom->data = data;  in mtk_iommu_v1_domain_finalise()
284 	return &dom->domain;  in mtk_iommu_v1_domain_alloc()
290 	struct mtk_iommu_v1_data *data = dom->data;  in mtk_iommu_v1_domain_free()
292 	dma_free_coherent(data->dev, M2701_IOMMU_PGT_SIZE,  in mtk_iommu_v1_domain_free()
293 			dom->pgt_va, dom->pgt_pa);  in mtk_iommu_v1_domain_free()
305 	mtk_mapping = data->mapping;  in mtk_iommu_v1_attach_device()
306 	if (mtk_mapping->domain != domain)  in mtk_iommu_v1_attach_device()
309 	if (!data->m4u_dom) {  in mtk_iommu_v1_attach_device()
310 		data->m4u_dom = dom;  in mtk_iommu_v1_attach_device()
313 			data->m4u_dom = NULL;  in mtk_iommu_v1_attach_device()
336 	u32 *pgt_base_iova = dom->pgt_va + (iova  >> MT2701_IOMMU_PAGE_SHIFT);  in mtk_iommu_v1_map()
340 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_map()
351 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_map()
353 	mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);  in mtk_iommu_v1_map()
355 	return map_size == size ? 0 : -EEXIST;  in mtk_iommu_v1_map()
363 	u32 *pgt_base_iova = dom->pgt_va + (iova  >> MT2701_IOMMU_PAGE_SHIFT);  in mtk_iommu_v1_unmap()
366 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_unmap()
368 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_unmap()
370 	mtk_iommu_v1_tlb_flush_range(dom->data, iova, size);  in mtk_iommu_v1_unmap()
381 	spin_lock_irqsave(&dom->pgtlock, flags);  in mtk_iommu_v1_iova_to_phys()
382 	pa = *(dom->pgt_va + (iova >> MT2701_IOMMU_PAGE_SHIFT));  in mtk_iommu_v1_iova_to_phys()
383 	pa = pa & (~(MT2701_IOMMU_PAGE_SIZE - 1));  in mtk_iommu_v1_iova_to_phys()
384 	spin_unlock_irqrestore(&dom->pgtlock, flags);  in mtk_iommu_v1_iova_to_phys()
403 	if (args->args_count != 1) {  in mtk_iommu_v1_create_mapping()
404 		dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n",  in mtk_iommu_v1_create_mapping()
405 			args->args_count);  in mtk_iommu_v1_create_mapping()
406 		return -EINVAL;  in mtk_iommu_v1_create_mapping()
410 		ret = iommu_fwspec_init(dev, &args->np->fwnode, &mtk_iommu_v1_ops);  in mtk_iommu_v1_create_mapping()
414 	} else if (dev_iommu_fwspec_get(dev)->ops != &mtk_iommu_v1_ops) {  in mtk_iommu_v1_create_mapping()
415 		return -EINVAL;  in mtk_iommu_v1_create_mapping()
420 		m4updev = of_find_device_by_node(args->np);  in mtk_iommu_v1_create_mapping()
422 			return -EINVAL;  in mtk_iommu_v1_create_mapping()
427 	ret = iommu_fwspec_add_ids(dev, args->args, 1);  in mtk_iommu_v1_create_mapping()
432 	mtk_mapping = data->mapping;  in mtk_iommu_v1_create_mapping()
440 		data->mapping = mtk_mapping;  in mtk_iommu_v1_create_mapping()
469 	while (!of_parse_phandle_with_args(dev->of_node, "iommus",  in mtk_iommu_v1_probe_device()
470 					   "#iommu-cells",  in mtk_iommu_v1_probe_device()
478 		/* dev->iommu_fwspec might have changed */  in mtk_iommu_v1_probe_device()
483 	if (!fwspec || fwspec->ops != &mtk_iommu_v1_ops)  in mtk_iommu_v1_probe_device()
484 		return ERR_PTR(-ENODEV); /* Not a iommu client device */  in mtk_iommu_v1_probe_device()
488 	/* Link the consumer device with the smi-larb device(supplier) */  in mtk_iommu_v1_probe_device()
489 	larbid = mt2701_m4u_to_larb(fwspec->ids[0]);  in mtk_iommu_v1_probe_device()
491 		return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
493 	for (idx = 1; idx < fwspec->num_ids; idx++) {  in mtk_iommu_v1_probe_device()
494 		larbidx = mt2701_m4u_to_larb(fwspec->ids[idx]);  in mtk_iommu_v1_probe_device()
496 			dev_err(dev, "Can only use one larb. Fail@larb%d-%d.\n",  in mtk_iommu_v1_probe_device()
498 			return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
502 	larbdev = data->larb_imu[larbid].dev;  in mtk_iommu_v1_probe_device()
504 		return ERR_PTR(-EINVAL);  in mtk_iommu_v1_probe_device()
511 	return &data->iommu;  in mtk_iommu_v1_probe_device()
521 	mtk_mapping = data->mapping;  in mtk_iommu_v1_probe_finalize()
525 		dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n");  in mtk_iommu_v1_probe_finalize()
536 	larbid = mt2701_m4u_to_larb(fwspec->ids[0]);  in mtk_iommu_v1_release_device()
537 	larbdev = data->larb_imu[larbid].dev;  in mtk_iommu_v1_release_device()
546 	ret = clk_prepare_enable(data->bclk);  in mtk_iommu_v1_hw_init()
548 		dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret);  in mtk_iommu_v1_hw_init()
553 	writel_relaxed(regval, data->base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_hw_init()
563 	writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_hw_init()
566 	writel_relaxed(data->protect_base,  in mtk_iommu_v1_hw_init()
567 			data->base + REG_MMU_IVRP_PADDR);  in mtk_iommu_v1_hw_init()
569 	writel_relaxed(F_MMU_DCM_ON, data->base + REG_MMU_DCM);  in mtk_iommu_v1_hw_init()
571 	if (devm_request_irq(data->dev, data->irq, mtk_iommu_v1_isr, 0,  in mtk_iommu_v1_hw_init()
572 			     dev_name(data->dev), (void *)data)) {  in mtk_iommu_v1_hw_init()
573 		writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_hw_init()
574 		clk_disable_unprepare(data->bclk);  in mtk_iommu_v1_hw_init()
575 		dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq);  in mtk_iommu_v1_hw_init()
576 		return -ENODEV;  in mtk_iommu_v1_hw_init()
602 	{ .compatible = "mediatek,mt2701-m4u", },
613 	struct device			*dev = &pdev->dev;  in mtk_iommu_v1_probe()
622 		return -ENOMEM;  in mtk_iommu_v1_probe()
624 	data->dev = dev;  in mtk_iommu_v1_probe()
630 		return -ENOMEM;  in mtk_iommu_v1_probe()
631 	data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN);  in mtk_iommu_v1_probe()
634 	data->base = devm_ioremap_resource(dev, res);  in mtk_iommu_v1_probe()
635 	if (IS_ERR(data->base))  in mtk_iommu_v1_probe()
636 		return PTR_ERR(data->base);  in mtk_iommu_v1_probe()
638 	data->irq = platform_get_irq(pdev, 0);  in mtk_iommu_v1_probe()
639 	if (data->irq < 0)  in mtk_iommu_v1_probe()
640 		return data->irq;  in mtk_iommu_v1_probe()
642 	data->bclk = devm_clk_get(dev, "bclk");  in mtk_iommu_v1_probe()
643 	if (IS_ERR(data->bclk))  in mtk_iommu_v1_probe()
644 		return PTR_ERR(data->bclk);  in mtk_iommu_v1_probe()
646 	larb_nr = of_count_phandle_with_args(dev->of_node,  in mtk_iommu_v1_probe()
655 		larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i);  in mtk_iommu_v1_probe()
657 			return -EINVAL;  in mtk_iommu_v1_probe()
667 			return -ENODEV;  in mtk_iommu_v1_probe()
669 		if (!plarbdev->dev.driver) {  in mtk_iommu_v1_probe()
671 			return -EPROBE_DEFER;  in mtk_iommu_v1_probe()
673 		data->larb_imu[i].dev = &plarbdev->dev;  in mtk_iommu_v1_probe()
685 	ret = iommu_device_sysfs_add(&data->iommu, &pdev->dev, NULL,  in mtk_iommu_v1_probe()
686 				     dev_name(&pdev->dev));  in mtk_iommu_v1_probe()
690 	ret = iommu_device_register(&data->iommu, &mtk_iommu_v1_ops, dev);  in mtk_iommu_v1_probe()
700 	iommu_device_unregister(&data->iommu);  in mtk_iommu_v1_probe()
702 	iommu_device_sysfs_remove(&data->iommu);  in mtk_iommu_v1_probe()
710 	iommu_device_sysfs_remove(&data->iommu);  in mtk_iommu_v1_remove()
711 	iommu_device_unregister(&data->iommu);  in mtk_iommu_v1_remove()
713 	clk_disable_unprepare(data->bclk);  in mtk_iommu_v1_remove()
714 	devm_free_irq(&pdev->dev, data->irq, data);  in mtk_iommu_v1_remove()
715 	component_master_del(&pdev->dev, &mtk_iommu_v1_com_ops);  in mtk_iommu_v1_remove()
722 	struct mtk_iommu_v1_suspend_reg *reg = &data->reg;  in mtk_iommu_v1_suspend()
723 	void __iomem *base = data->base;  in mtk_iommu_v1_suspend()
725 	reg->standard_axi_mode = readl_relaxed(base +  in mtk_iommu_v1_suspend()
727 	reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM);  in mtk_iommu_v1_suspend()
728 	reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_suspend()
729 	reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_suspend()
736 	struct mtk_iommu_v1_suspend_reg *reg = &data->reg;  in mtk_iommu_v1_resume()
737 	void __iomem *base = data->base;  in mtk_iommu_v1_resume()
739 	writel_relaxed(data->m4u_dom->pgt_pa, base + REG_MMU_PT_BASE_ADDR);  in mtk_iommu_v1_resume()
740 	writel_relaxed(reg->standard_axi_mode,  in mtk_iommu_v1_resume()
742 	writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM);  in mtk_iommu_v1_resume()
743 	writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG);  in mtk_iommu_v1_resume()
744 	writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL);  in mtk_iommu_v1_resume()
745 	writel_relaxed(data->protect_base, base + REG_MMU_IVRP_PADDR);  in mtk_iommu_v1_resume()
757 		.name = "mtk-iommu-v1",