Lines Matching +full:secure +full:- +full:reg +full:- +full:access
1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for Renesas VMSA-compatible IPMMU
11 #include <linux/dma-iommu.h>
12 #include <linux/dma-mapping.h>
18 #include <linux/io-pgtable.h>
30 #include <asm/dma-iommu.h>
34 #define arm_iommu_attach_device(...) -ENODEV
40 #define IPMMU_CTX_INVALID -1
91 return fwspec ? fwspec->iommu_priv : NULL; in to_ipmmu()
96 /* -----------------------------------------------------------------------------
119 #define IMTTBCR_SH1_NON_SHAREABLE (0 << 28) /* R-Car Gen2 only */
120 #define IMTTBCR_SH1_OUTER_SHAREABLE (2 << 28) /* R-Car Gen2 only */
121 #define IMTTBCR_SH1_INNER_SHAREABLE (3 << 28) /* R-Car Gen2 only */
122 #define IMTTBCR_SH1_MASK (3 << 28) /* R-Car Gen2 only */
123 #define IMTTBCR_ORGN1_NC (0 << 26) /* R-Car Gen2 only */
124 #define IMTTBCR_ORGN1_WB_WA (1 << 26) /* R-Car Gen2 only */
125 #define IMTTBCR_ORGN1_WT (2 << 26) /* R-Car Gen2 only */
126 #define IMTTBCR_ORGN1_WB (3 << 26) /* R-Car Gen2 only */
127 #define IMTTBCR_ORGN1_MASK (3 << 26) /* R-Car Gen2 only */
128 #define IMTTBCR_IRGN1_NC (0 << 24) /* R-Car Gen2 only */
129 #define IMTTBCR_IRGN1_WB_WA (1 << 24) /* R-Car Gen2 only */
130 #define IMTTBCR_IRGN1_WT (2 << 24) /* R-Car Gen2 only */
131 #define IMTTBCR_IRGN1_WB (3 << 24) /* R-Car Gen2 only */
132 #define IMTTBCR_IRGN1_MASK (3 << 24) /* R-Car Gen2 only */
135 #define IMTTBCR_SH0_NON_SHAREABLE (0 << 12) /* R-Car Gen2 only */
136 #define IMTTBCR_SH0_OUTER_SHAREABLE (2 << 12) /* R-Car Gen2 only */
137 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */
138 #define IMTTBCR_SH0_MASK (3 << 12) /* R-Car Gen2 only */
139 #define IMTTBCR_ORGN0_NC (0 << 10) /* R-Car Gen2 only */
140 #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */
141 #define IMTTBCR_ORGN0_WT (2 << 10) /* R-Car Gen2 only */
142 #define IMTTBCR_ORGN0_WB (3 << 10) /* R-Car Gen2 only */
143 #define IMTTBCR_ORGN0_MASK (3 << 10) /* R-Car Gen2 only */
144 #define IMTTBCR_IRGN0_NC (0 << 8) /* R-Car Gen2 only */
145 #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */
146 #define IMTTBCR_IRGN0_WT (2 << 8) /* R-Car Gen2 only */
147 #define IMTTBCR_IRGN0_WB (3 << 8) /* R-Car Gen2 only */
148 #define IMTTBCR_IRGN0_MASK (3 << 8) /* R-Car Gen2 only */
149 #define IMTTBCR_SL0_TWOBIT_LVL_3 (0 << 6) /* R-Car Gen3 only */
150 #define IMTTBCR_SL0_TWOBIT_LVL_2 (1 << 6) /* R-Car Gen3 only */
151 #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */
193 #define IMELAR 0x0030 /* IMEAR on R-Car Gen2 */
194 #define IMEUAR 0x0034 /* R-Car Gen3 only */
204 #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16))
216 #define IMUASID32(n) (0x0608 + (((n) - 32) * 16))
222 /* -----------------------------------------------------------------------------
230 return mmu->root == mmu; in ipmmu_is_root()
252 /* -----------------------------------------------------------------------------
253 * Read/Write Access
258 return ioread32(mmu->base + offset); in ipmmu_read()
264 iowrite32(data, mmu->base + offset); in ipmmu_write()
268 unsigned int reg) in ipmmu_ctx_read_root() argument
270 return ipmmu_read(domain->mmu->root, in ipmmu_ctx_read_root()
271 domain->context_id * IM_CTX_SIZE + reg); in ipmmu_ctx_read_root()
275 unsigned int reg, u32 data) in ipmmu_ctx_write_root() argument
277 ipmmu_write(domain->mmu->root, in ipmmu_ctx_write_root()
278 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_root()
282 unsigned int reg, u32 data) in ipmmu_ctx_write_all() argument
284 if (domain->mmu != domain->mmu->root) in ipmmu_ctx_write_all()
285 ipmmu_write(domain->mmu, in ipmmu_ctx_write_all()
286 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_all()
288 ipmmu_write(domain->mmu->root, in ipmmu_ctx_write_all()
289 domain->context_id * IM_CTX_SIZE + reg, data); in ipmmu_ctx_write_all()
292 /* -----------------------------------------------------------------------------
304 dev_err_ratelimited(domain->mmu->dev, in ipmmu_tlb_sync()
305 "TLB sync timed out -- MMU may be deadlocked\n"); in ipmmu_tlb_sync()
314 u32 reg; in ipmmu_tlb_invalidate() local
316 reg = ipmmu_ctx_read_root(domain, IMCTR); in ipmmu_tlb_invalidate()
317 reg |= IMCTR_FLUSH; in ipmmu_tlb_invalidate()
318 ipmmu_ctx_write_all(domain, IMCTR, reg); in ipmmu_tlb_invalidate()
329 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_enable()
332 * TODO: Reference-count the microTLB as several bus masters can be in ipmmu_utlb_enable()
340 IMUCTR_TTSEL_MMU(domain->context_id) | IMUCTR_FLUSH | in ipmmu_utlb_enable()
342 mmu->utlb_ctx[utlb] = domain->context_id; in ipmmu_utlb_enable()
351 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_disable()
354 mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; in ipmmu_utlb_disable()
376 /* -----------------------------------------------------------------------------
386 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_allocate_context()
388 ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); in ipmmu_domain_allocate_context()
389 if (ret != mmu->num_ctx) { in ipmmu_domain_allocate_context()
390 mmu->domains[ret] = domain; in ipmmu_domain_allocate_context()
391 set_bit(ret, mmu->ctx); in ipmmu_domain_allocate_context()
393 ret = -EBUSY; in ipmmu_domain_allocate_context()
395 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_allocate_context()
405 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_free_context()
407 clear_bit(context_id, mmu->ctx); in ipmmu_domain_free_context()
408 mmu->domains[context_id] = NULL; in ipmmu_domain_free_context()
410 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_free_context()
419 ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr[0]; in ipmmu_domain_setup_context()
425 * We use long descriptors and allocate the whole 32-bit VA space to in ipmmu_domain_setup_context()
428 if (domain->mmu->features->twobit_imttbcr_sl0) in ipmmu_domain_setup_context()
433 if (domain->mmu->features->cache_snoop) in ipmmu_domain_setup_context()
441 domain->cfg.arm_lpae_s1_cfg.mair[0]); in ipmmu_domain_setup_context()
444 if (domain->mmu->features->setup_imbuscr) in ipmmu_domain_setup_context()
457 * Enable the MMU and interrupt generation. The long-descriptor in ipmmu_domain_setup_context()
473 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory in ipmmu_domain_init_context()
474 * access, Long-descriptor format" that the NStable bit being set in a in ipmmu_domain_init_context()
477 * not to comply with this, as it generates a secure access page fault in ipmmu_domain_init_context()
479 * non-secure mode. in ipmmu_domain_init_context()
481 domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; in ipmmu_domain_init_context()
482 domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; in ipmmu_domain_init_context()
483 domain->cfg.ias = 32; in ipmmu_domain_init_context()
484 domain->cfg.oas = 40; in ipmmu_domain_init_context()
485 domain->cfg.tlb = &ipmmu_flush_ops; in ipmmu_domain_init_context()
486 domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); in ipmmu_domain_init_context()
487 domain->io_domain.geometry.force_aperture = true; in ipmmu_domain_init_context()
490 * cache handling. For now, delegate it to the io-pgtable code. in ipmmu_domain_init_context()
492 domain->cfg.coherent_walk = false; in ipmmu_domain_init_context()
493 domain->cfg.iommu_dev = domain->mmu->root->dev; in ipmmu_domain_init_context()
498 ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); in ipmmu_domain_init_context()
502 domain->context_id = ret; in ipmmu_domain_init_context()
504 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, in ipmmu_domain_init_context()
506 if (!domain->iop) { in ipmmu_domain_init_context()
507 ipmmu_domain_free_context(domain->mmu->root, in ipmmu_domain_init_context()
508 domain->context_id); in ipmmu_domain_init_context()
509 return -EINVAL; in ipmmu_domain_init_context()
518 if (!domain->mmu) in ipmmu_domain_destroy_context()
529 ipmmu_domain_free_context(domain->mmu->root, domain->context_id); in ipmmu_domain_destroy_context()
532 /* -----------------------------------------------------------------------------
539 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_domain_irq()
561 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n", in ipmmu_domain_irq()
564 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n", in ipmmu_domain_irq()
576 if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) in ipmmu_domain_irq()
579 dev_err_ratelimited(mmu->dev, in ipmmu_domain_irq()
593 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_irq()
598 for (i = 0; i < mmu->num_ctx; i++) { in ipmmu_irq()
599 if (!mmu->domains[i]) in ipmmu_irq()
601 if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) in ipmmu_irq()
605 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_irq()
610 /* -----------------------------------------------------------------------------
622 mutex_init(&domain->mutex); in __ipmmu_domain_alloc()
624 return &domain->io_domain; in __ipmmu_domain_alloc()
658 free_io_pgtable_ops(domain->iop); in ipmmu_domain_free()
673 return -ENXIO; in ipmmu_attach_device()
676 mutex_lock(&domain->mutex); in ipmmu_attach_device()
678 if (!domain->mmu) { in ipmmu_attach_device()
680 domain->mmu = mmu; in ipmmu_attach_device()
684 domain->mmu = NULL; in ipmmu_attach_device()
687 domain->context_id); in ipmmu_attach_device()
689 } else if (domain->mmu != mmu) { in ipmmu_attach_device()
695 dev_name(mmu->dev), dev_name(domain->mmu->dev)); in ipmmu_attach_device()
696 ret = -EINVAL; in ipmmu_attach_device()
698 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); in ipmmu_attach_device()
700 mutex_unlock(&domain->mutex); in ipmmu_attach_device()
705 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_attach_device()
706 ipmmu_utlb_enable(domain, fwspec->ids[i]); in ipmmu_attach_device()
718 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_detach_device()
719 ipmmu_utlb_disable(domain, fwspec->ids[i]); in ipmmu_detach_device()
732 return -ENODEV; in ipmmu_map()
734 return domain->iop->map(domain->iop, iova, paddr, size, prot); in ipmmu_map()
742 return domain->iop->unmap(domain->iop, iova, size, gather); in ipmmu_unmap()
749 if (domain->mmu) in ipmmu_flush_iotlb_all()
766 return domain->iop->iova_to_phys(domain->iop, iova); in ipmmu_iova_to_phys()
775 ipmmu_pdev = of_find_device_by_node(args->np); in ipmmu_init_platform_device()
777 return -ENODEV; in ipmmu_init_platform_device()
779 fwspec->iommu_priv = platform_get_drvdata(ipmmu_pdev); in ipmmu_init_platform_device()
813 * For R-Car Gen3 use a white list to opt-in slave devices. in ipmmu_slave_whitelist()
819 /* Check whether this R-Car Gen3 can use the IPMMU correctly or not */ in ipmmu_slave_whitelist()
837 return -ENODEV; in ipmmu_of_xlate()
839 iommu_fwspec_add_ids(dev, spec->args, 1); in ipmmu_of_xlate()
841 /* Initialize once - xlate() will call multiple times */ in ipmmu_of_xlate()
874 * - Create one mapping per context (TLB). in ipmmu_init_arm_mapping()
875 * - Make the mapping size configurable ? We currently use a 2GB mapping in ipmmu_init_arm_mapping()
878 if (!mmu->mapping) { in ipmmu_init_arm_mapping()
884 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); in ipmmu_init_arm_mapping()
889 mmu->mapping = mapping; in ipmmu_init_arm_mapping()
893 ret = arm_iommu_attach_device(dev, mmu->mapping); in ipmmu_init_arm_mapping()
903 if (mmu->mapping) in ipmmu_init_arm_mapping()
904 arm_iommu_release_mapping(mmu->mapping); in ipmmu_init_arm_mapping()
919 return -ENODEV; in ipmmu_add_device()
933 iommu_device_link(&mmu->iommu, dev); in ipmmu_add_device()
941 iommu_device_unlink(&mmu->iommu, dev); in ipmmu_remove_device()
951 if (mmu->group) in ipmmu_find_group()
952 return iommu_group_ref_get(mmu->group); in ipmmu_find_group()
956 mmu->group = group; in ipmmu_find_group()
978 /* -----------------------------------------------------------------------------
987 for (i = 0; i < mmu->num_ctx; ++i) in ipmmu_device_reset()
1015 .compatible = "renesas,ipmmu-vmsa",
1018 .compatible = "renesas,ipmmu-r8a774a1",
1021 .compatible = "renesas,ipmmu-r8a774c0",
1024 .compatible = "renesas,ipmmu-r8a7795",
1027 .compatible = "renesas,ipmmu-r8a7796",
1030 .compatible = "renesas,ipmmu-r8a77965",
1033 .compatible = "renesas,ipmmu-r8a77970",
1036 .compatible = "renesas,ipmmu-r8a77990",
1039 .compatible = "renesas,ipmmu-r8a77995",
1053 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); in ipmmu_probe()
1055 dev_err(&pdev->dev, "cannot allocate device data\n"); in ipmmu_probe()
1056 return -ENOMEM; in ipmmu_probe()
1059 mmu->dev = &pdev->dev; in ipmmu_probe()
1060 spin_lock_init(&mmu->lock); in ipmmu_probe()
1061 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); in ipmmu_probe()
1062 mmu->features = of_device_get_match_data(&pdev->dev); in ipmmu_probe()
1063 memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); in ipmmu_probe()
1064 dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); in ipmmu_probe()
1068 mmu->base = devm_ioremap_resource(&pdev->dev, res); in ipmmu_probe()
1069 if (IS_ERR(mmu->base)) in ipmmu_probe()
1070 return PTR_ERR(mmu->base); in ipmmu_probe()
1073 * The IPMMU has two register banks, for secure and non-secure modes. in ipmmu_probe()
1075 * corresponds to the running mode of the CPU. When running in secure in ipmmu_probe()
1076 * mode the non-secure register bank is also available at an offset. in ipmmu_probe()
1078 * Secure mode operation isn't clearly documented and is thus currently in ipmmu_probe()
1080 * non-secure operation with the main register bank were not successful. in ipmmu_probe()
1081 * Offset the registers base unconditionally to point to the non-secure in ipmmu_probe()
1084 if (mmu->features->use_ns_alias_offset) in ipmmu_probe()
1085 mmu->base += IM_NS_ALIAS_OFFSET; in ipmmu_probe()
1087 mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); in ipmmu_probe()
1091 * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. in ipmmu_probe()
1093 if (!mmu->features->has_cache_leaf_nodes || in ipmmu_probe()
1094 !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) in ipmmu_probe()
1095 mmu->root = mmu; in ipmmu_probe()
1097 mmu->root = ipmmu_find_root(); in ipmmu_probe()
1102 if (!mmu->root) in ipmmu_probe()
1103 return -EPROBE_DEFER; in ipmmu_probe()
1111 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, in ipmmu_probe()
1112 dev_name(&pdev->dev), mmu); in ipmmu_probe()
1114 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); in ipmmu_probe()
1120 if (mmu->features->reserved_context) { in ipmmu_probe()
1121 dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); in ipmmu_probe()
1122 set_bit(0, mmu->ctx); in ipmmu_probe()
1128 * - R-Car Gen2 IPMMU (all devices registered) in ipmmu_probe()
1129 * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) in ipmmu_probe()
1131 if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { in ipmmu_probe()
1132 ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, in ipmmu_probe()
1133 dev_name(&pdev->dev)); in ipmmu_probe()
1137 iommu_device_set_ops(&mmu->iommu, &ipmmu_ops); in ipmmu_probe()
1138 iommu_device_set_fwnode(&mmu->iommu, in ipmmu_probe()
1139 &pdev->dev.of_node->fwnode); in ipmmu_probe()
1141 ret = iommu_device_register(&mmu->iommu); in ipmmu_probe()
1166 iommu_device_sysfs_remove(&mmu->iommu); in ipmmu_remove()
1167 iommu_device_unregister(&mmu->iommu); in ipmmu_remove()
1169 arm_iommu_release_mapping(mmu->mapping); in ipmmu_remove()
1186 for (i = 0; i < mmu->num_ctx; i++) { in ipmmu_resume_noirq()
1187 if (!mmu->domains[i]) in ipmmu_resume_noirq()
1190 ipmmu_domain_setup_context(mmu->domains[i]); in ipmmu_resume_noirq()
1194 /* Re-enable active micro-TLBs */ in ipmmu_resume_noirq()
1195 for (i = 0; i < mmu->features->num_utlbs; i++) { in ipmmu_resume_noirq()
1196 if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) in ipmmu_resume_noirq()
1199 ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i); in ipmmu_resume_noirq()
1215 .name = "ipmmu-vmsa",