Lines Matching +full:ipmmu +full:- +full:vmsa
1 // SPDX-License-Identifier: GPL-2.0
3 * IOMMU API for Renesas VMSA-compatible IPMMU
6 * Copyright (C) 2014-2020 Renesas Electronics Corporation
11 #include <linux/dma-mapping.h>
17 #include <linux/io-pgtable.h>
28 #include <asm/dma-iommu.h>
31 #define arm_iommu_attach_device(...) -ENODEV
37 #define IPMMU_CTX_INVALID -1
94 /* -----------------------------------------------------------------------------
101 #define IMCTR 0x0000 /* R-Car Gen2/3 */
102 #define IMCTR_INTEN (1 << 2) /* R-Car Gen2/3 */
103 #define IMCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
104 #define IMCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
106 #define IMTTBCR 0x0008 /* R-Car Gen2/3 */
107 #define IMTTBCR_EAE (1 << 31) /* R-Car Gen2/3 */
108 #define IMTTBCR_SH0_INNER_SHAREABLE (3 << 12) /* R-Car Gen2 only */
109 #define IMTTBCR_ORGN0_WB_WA (1 << 10) /* R-Car Gen2 only */
110 #define IMTTBCR_IRGN0_WB_WA (1 << 8) /* R-Car Gen2 only */
111 #define IMTTBCR_SL0_TWOBIT_LVL_1 (2 << 6) /* R-Car Gen3 only */
112 #define IMTTBCR_SL0_LVL_1 (1 << 4) /* R-Car Gen2 only */
114 #define IMBUSCR 0x000c /* R-Car Gen2 only */
115 #define IMBUSCR_DVM (1 << 2) /* R-Car Gen2 only */
116 #define IMBUSCR_BUSSEL_MASK (3 << 0) /* R-Car Gen2 only */
118 #define IMTTLBR0 0x0010 /* R-Car Gen2/3 */
119 #define IMTTUBR0 0x0014 /* R-Car Gen2/3 */
121 #define IMSTR 0x0020 /* R-Car Gen2/3 */
122 #define IMSTR_MHIT (1 << 4) /* R-Car Gen2/3 */
123 #define IMSTR_ABORT (1 << 2) /* R-Car Gen2/3 */
124 #define IMSTR_PF (1 << 1) /* R-Car Gen2/3 */
125 #define IMSTR_TF (1 << 0) /* R-Car Gen2/3 */
127 #define IMMAIR0 0x0028 /* R-Car Gen2/3 */
129 #define IMELAR 0x0030 /* R-Car Gen2/3, IMEAR on R-Car Gen2 */
130 #define IMEUAR 0x0034 /* R-Car Gen3 only */
134 #define IMUCTR0(n) (0x0300 + ((n) * 16)) /* R-Car Gen2/3 */
135 #define IMUCTR32(n) (0x0600 + (((n) - 32) * 16)) /* R-Car Gen3 only */
136 #define IMUCTR_TTSEL_MMU(n) ((n) << 4) /* R-Car Gen2/3 */
137 #define IMUCTR_FLUSH (1 << 1) /* R-Car Gen2/3 */
138 #define IMUCTR_MMUEN (1 << 0) /* R-Car Gen2/3 */
141 #define IMUASID0(n) (0x0308 + ((n) * 16)) /* R-Car Gen2/3 */
142 #define IMUASID32(n) (0x0608 + (((n) - 32) * 16)) /* R-Car Gen3 only */
144 /* -----------------------------------------------------------------------------
152 return mmu->root == mmu; in ipmmu_is_root()
174 /* -----------------------------------------------------------------------------
180 return ioread32(mmu->base + offset); in ipmmu_read()
186 iowrite32(data, mmu->base + offset); in ipmmu_write()
192 unsigned int base = mmu->features->ctx_offset_base; in ipmmu_ctx_reg()
195 base += 0x800 - 8 * 0x40; in ipmmu_ctx_reg()
197 return base + context_id * mmu->features->ctx_offset_stride + reg; in ipmmu_ctx_reg()
215 return ipmmu_ctx_read(domain->mmu->root, domain->context_id, reg); in ipmmu_ctx_read_root()
221 ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data); in ipmmu_ctx_write_root()
227 if (domain->mmu != domain->mmu->root) in ipmmu_ctx_write_all()
228 ipmmu_ctx_write(domain->mmu, domain->context_id, reg, data); in ipmmu_ctx_write_all()
230 ipmmu_ctx_write(domain->mmu->root, domain->context_id, reg, data); in ipmmu_ctx_write_all()
235 return mmu->features->utlb_offset_base + reg; in ipmmu_utlb_reg()
250 /* -----------------------------------------------------------------------------
262 dev_err_ratelimited(domain->mmu->dev, in ipmmu_tlb_sync()
263 "TLB sync timed out -- MMU may be deadlocked\n"); in ipmmu_tlb_sync()
287 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_enable()
290 * TODO: Reference-count the microTLB as several bus masters can be in ipmmu_utlb_enable()
297 ipmmu_imuctr_write(mmu, utlb, IMUCTR_TTSEL_MMU(domain->context_id) | in ipmmu_utlb_enable()
299 mmu->utlb_ctx[utlb] = domain->context_id; in ipmmu_utlb_enable()
308 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_utlb_disable()
311 mmu->utlb_ctx[utlb] = IPMMU_CTX_INVALID; in ipmmu_utlb_disable()
332 /* -----------------------------------------------------------------------------
342 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_allocate_context()
344 ret = find_first_zero_bit(mmu->ctx, mmu->num_ctx); in ipmmu_domain_allocate_context()
345 if (ret != mmu->num_ctx) { in ipmmu_domain_allocate_context()
346 mmu->domains[ret] = domain; in ipmmu_domain_allocate_context()
347 set_bit(ret, mmu->ctx); in ipmmu_domain_allocate_context()
349 ret = -EBUSY; in ipmmu_domain_allocate_context()
351 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_allocate_context()
361 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_domain_free_context()
363 clear_bit(context_id, mmu->ctx); in ipmmu_domain_free_context()
364 mmu->domains[context_id] = NULL; in ipmmu_domain_free_context()
366 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_domain_free_context()
375 ttbr = domain->cfg.arm_lpae_s1_cfg.ttbr; in ipmmu_domain_setup_context()
381 * We use long descriptors and allocate the whole 32-bit VA space to in ipmmu_domain_setup_context()
384 if (domain->mmu->features->twobit_imttbcr_sl0) in ipmmu_domain_setup_context()
389 if (domain->mmu->features->cache_snoop) in ipmmu_domain_setup_context()
397 domain->cfg.arm_lpae_s1_cfg.mair); in ipmmu_domain_setup_context()
400 if (domain->mmu->features->setup_imbuscr) in ipmmu_domain_setup_context()
413 * Enable the MMU and interrupt generation. The long-descriptor in ipmmu_domain_setup_context()
429 * VMSA states in section B3.6.3 "Control of Secure or Non-secure memory in ipmmu_domain_init_context()
430 * access, Long-descriptor format" that the NStable bit being set in a in ipmmu_domain_init_context()
432 * entries being ignored and considered as being set. The IPMMU seems in ipmmu_domain_init_context()
435 * non-secure mode. in ipmmu_domain_init_context()
437 domain->cfg.quirks = IO_PGTABLE_QUIRK_ARM_NS; in ipmmu_domain_init_context()
438 domain->cfg.pgsize_bitmap = SZ_1G | SZ_2M | SZ_4K; in ipmmu_domain_init_context()
439 domain->cfg.ias = 32; in ipmmu_domain_init_context()
440 domain->cfg.oas = 40; in ipmmu_domain_init_context()
441 domain->cfg.tlb = &ipmmu_flush_ops; in ipmmu_domain_init_context()
442 domain->io_domain.geometry.aperture_end = DMA_BIT_MASK(32); in ipmmu_domain_init_context()
443 domain->io_domain.geometry.force_aperture = true; in ipmmu_domain_init_context()
446 * cache handling. For now, delegate it to the io-pgtable code. in ipmmu_domain_init_context()
448 domain->cfg.coherent_walk = false; in ipmmu_domain_init_context()
449 domain->cfg.iommu_dev = domain->mmu->root->dev; in ipmmu_domain_init_context()
454 ret = ipmmu_domain_allocate_context(domain->mmu->root, domain); in ipmmu_domain_init_context()
458 domain->context_id = ret; in ipmmu_domain_init_context()
460 domain->iop = alloc_io_pgtable_ops(ARM_32_LPAE_S1, &domain->cfg, in ipmmu_domain_init_context()
462 if (!domain->iop) { in ipmmu_domain_init_context()
463 ipmmu_domain_free_context(domain->mmu->root, in ipmmu_domain_init_context()
464 domain->context_id); in ipmmu_domain_init_context()
465 return -EINVAL; in ipmmu_domain_init_context()
474 if (!domain->mmu) in ipmmu_domain_destroy_context()
485 ipmmu_domain_free_context(domain->mmu->root, domain->context_id); in ipmmu_domain_destroy_context()
488 /* -----------------------------------------------------------------------------
495 struct ipmmu_vmsa_device *mmu = domain->mmu; in ipmmu_domain_irq()
517 dev_err_ratelimited(mmu->dev, "Multiple TLB hits @0x%lx\n", in ipmmu_domain_irq()
520 dev_err_ratelimited(mmu->dev, "Page Table Walk Abort @0x%lx\n", in ipmmu_domain_irq()
532 if (!report_iommu_fault(&domain->io_domain, mmu->dev, iova, 0)) in ipmmu_domain_irq()
535 dev_err_ratelimited(mmu->dev, in ipmmu_domain_irq()
549 spin_lock_irqsave(&mmu->lock, flags); in ipmmu_irq()
554 for (i = 0; i < mmu->num_ctx; i++) { in ipmmu_irq()
555 if (!mmu->domains[i]) in ipmmu_irq()
557 if (ipmmu_domain_irq(mmu->domains[i]) == IRQ_HANDLED) in ipmmu_irq()
561 spin_unlock_irqrestore(&mmu->lock, flags); in ipmmu_irq()
566 /* -----------------------------------------------------------------------------
581 mutex_init(&domain->mutex); in ipmmu_domain_alloc()
583 return &domain->io_domain; in ipmmu_domain_alloc()
595 free_io_pgtable_ops(domain->iop); in ipmmu_domain_free()
609 dev_err(dev, "Cannot attach to IPMMU\n"); in ipmmu_attach_device()
610 return -ENXIO; in ipmmu_attach_device()
613 mutex_lock(&domain->mutex); in ipmmu_attach_device()
615 if (!domain->mmu) { in ipmmu_attach_device()
617 domain->mmu = mmu; in ipmmu_attach_device()
620 dev_err(dev, "Unable to initialize IPMMU context\n"); in ipmmu_attach_device()
621 domain->mmu = NULL; in ipmmu_attach_device()
623 dev_info(dev, "Using IPMMU context %u\n", in ipmmu_attach_device()
624 domain->context_id); in ipmmu_attach_device()
626 } else if (domain->mmu != mmu) { in ipmmu_attach_device()
631 dev_err(dev, "Can't attach IPMMU %s to domain on IPMMU %s\n", in ipmmu_attach_device()
632 dev_name(mmu->dev), dev_name(domain->mmu->dev)); in ipmmu_attach_device()
633 ret = -EINVAL; in ipmmu_attach_device()
635 dev_info(dev, "Reusing IPMMU context %u\n", domain->context_id); in ipmmu_attach_device()
637 mutex_unlock(&domain->mutex); in ipmmu_attach_device()
642 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_attach_device()
643 ipmmu_utlb_enable(domain, fwspec->ids[i]); in ipmmu_attach_device()
655 for (i = 0; i < fwspec->num_ids; ++i) in ipmmu_detach_device()
656 ipmmu_utlb_disable(domain, fwspec->ids[i]); in ipmmu_detach_device()
669 return -ENODEV; in ipmmu_map()
671 return domain->iop->map(domain->iop, iova, paddr, size, prot, gfp); in ipmmu_map()
679 return domain->iop->unmap(domain->iop, iova, size, gather); in ipmmu_unmap()
686 if (domain->mmu) in ipmmu_flush_iotlb_all()
703 return domain->iop->iova_to_phys(domain->iop, iova); in ipmmu_iova_to_phys()
711 ipmmu_pdev = of_find_device_by_node(args->np); in ipmmu_init_platform_device()
713 return -ENODEV; in ipmmu_init_platform_device()
721 { .family = "R-Car Gen3", },
722 { .family = "R-Car Gen4", },
747 * R-Car Gen3/4 and RZ/G2 use the allow list to opt-in devices. in ipmmu_device_is_allowed()
753 /* Check whether this SoC can use the IPMMU correctly or not */ in ipmmu_device_is_allowed()
757 /* Check whether this device can work with the IPMMU */ in ipmmu_device_is_allowed()
763 /* Otherwise, do not allow use of IPMMU */ in ipmmu_device_is_allowed()
771 return -ENODEV; in ipmmu_of_xlate()
773 iommu_fwspec_add_ids(dev, spec->args, 1); in ipmmu_of_xlate()
775 /* Initialize once - xlate() will call multiple times */ in ipmmu_of_xlate()
792 * - Create one mapping per context (TLB). in ipmmu_init_arm_mapping()
793 * - Make the mapping size configurable ? We currently use a 2GB mapping in ipmmu_init_arm_mapping()
796 if (!mmu->mapping) { in ipmmu_init_arm_mapping()
802 dev_err(mmu->dev, "failed to create ARM IOMMU mapping\n"); in ipmmu_init_arm_mapping()
807 mmu->mapping = mapping; in ipmmu_init_arm_mapping()
811 ret = arm_iommu_attach_device(dev, mmu->mapping); in ipmmu_init_arm_mapping()
820 if (mmu->mapping) in ipmmu_init_arm_mapping()
821 arm_iommu_release_mapping(mmu->mapping); in ipmmu_init_arm_mapping()
834 return ERR_PTR(-ENODEV); in ipmmu_probe_device()
836 return &mmu->iommu; in ipmmu_probe_device()
847 dev_err(dev, "Can't create IOMMU mapping - DMA-OPS will not work\n"); in ipmmu_probe_finalize()
860 if (mmu->group) in ipmmu_find_group()
861 return iommu_group_ref_get(mmu->group); in ipmmu_find_group()
865 mmu->group = group; in ipmmu_find_group()
891 /* -----------------------------------------------------------------------------
900 for (i = 0; i < mmu->num_ctx; ++i) in ipmmu_device_reset()
948 .compatible = "renesas,ipmmu-vmsa",
951 .compatible = "renesas,ipmmu-r8a774a1",
954 .compatible = "renesas,ipmmu-r8a774b1",
957 .compatible = "renesas,ipmmu-r8a774c0",
960 .compatible = "renesas,ipmmu-r8a774e1",
963 .compatible = "renesas,ipmmu-r8a7795",
966 .compatible = "renesas,ipmmu-r8a7796",
969 .compatible = "renesas,ipmmu-r8a77961",
972 .compatible = "renesas,ipmmu-r8a77965",
975 .compatible = "renesas,ipmmu-r8a77970",
978 .compatible = "renesas,ipmmu-r8a77980",
981 .compatible = "renesas,ipmmu-r8a77990",
984 .compatible = "renesas,ipmmu-r8a77995",
987 .compatible = "renesas,ipmmu-r8a779a0",
990 .compatible = "renesas,rcar-gen4-ipmmu-vmsa",
1004 mmu = devm_kzalloc(&pdev->dev, sizeof(*mmu), GFP_KERNEL); in ipmmu_probe()
1006 dev_err(&pdev->dev, "cannot allocate device data\n"); in ipmmu_probe()
1007 return -ENOMEM; in ipmmu_probe()
1010 mmu->dev = &pdev->dev; in ipmmu_probe()
1011 spin_lock_init(&mmu->lock); in ipmmu_probe()
1012 bitmap_zero(mmu->ctx, IPMMU_CTX_MAX); in ipmmu_probe()
1013 mmu->features = of_device_get_match_data(&pdev->dev); in ipmmu_probe()
1014 memset(mmu->utlb_ctx, IPMMU_CTX_INVALID, mmu->features->num_utlbs); in ipmmu_probe()
1015 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40)); in ipmmu_probe()
1021 mmu->base = devm_ioremap_resource(&pdev->dev, res); in ipmmu_probe()
1022 if (IS_ERR(mmu->base)) in ipmmu_probe()
1023 return PTR_ERR(mmu->base); in ipmmu_probe()
1026 * The IPMMU has two register banks, for secure and non-secure modes. in ipmmu_probe()
1027 * The bank mapped at the beginning of the IPMMU address space in ipmmu_probe()
1029 * mode the non-secure register bank is also available at an offset. in ipmmu_probe()
1033 * non-secure operation with the main register bank were not successful. in ipmmu_probe()
1034 * Offset the registers base unconditionally to point to the non-secure in ipmmu_probe()
1037 if (mmu->features->use_ns_alias_offset) in ipmmu_probe()
1038 mmu->base += IM_NS_ALIAS_OFFSET; in ipmmu_probe()
1040 mmu->num_ctx = min(IPMMU_CTX_MAX, mmu->features->number_of_contexts); in ipmmu_probe()
1043 * Determine if this IPMMU instance is a root device by checking for in ipmmu_probe()
1044 * the lack of has_cache_leaf_nodes flag or renesas,ipmmu-main property. in ipmmu_probe()
1046 if (!mmu->features->has_cache_leaf_nodes || in ipmmu_probe()
1047 !of_find_property(pdev->dev.of_node, "renesas,ipmmu-main", NULL)) in ipmmu_probe()
1048 mmu->root = mmu; in ipmmu_probe()
1050 mmu->root = ipmmu_find_root(); in ipmmu_probe()
1055 if (!mmu->root) in ipmmu_probe()
1056 return -EPROBE_DEFER; in ipmmu_probe()
1064 ret = devm_request_irq(&pdev->dev, irq, ipmmu_irq, 0, in ipmmu_probe()
1065 dev_name(&pdev->dev), mmu); in ipmmu_probe()
1067 dev_err(&pdev->dev, "failed to request IRQ %d\n", irq); in ipmmu_probe()
1073 if (mmu->features->reserved_context) { in ipmmu_probe()
1074 dev_info(&pdev->dev, "IPMMU context 0 is reserved\n"); in ipmmu_probe()
1075 set_bit(0, mmu->ctx); in ipmmu_probe()
1080 * Register the IPMMU to the IOMMU subsystem in the following cases: in ipmmu_probe()
1081 * - R-Car Gen2 IPMMU (all devices registered) in ipmmu_probe()
1082 * - R-Car Gen3 IPMMU (leaf devices only - skip root IPMMU-MM device) in ipmmu_probe()
1084 if (!mmu->features->has_cache_leaf_nodes || !ipmmu_is_root(mmu)) { in ipmmu_probe()
1085 ret = iommu_device_sysfs_add(&mmu->iommu, &pdev->dev, NULL, in ipmmu_probe()
1086 dev_name(&pdev->dev)); in ipmmu_probe()
1090 ret = iommu_device_register(&mmu->iommu, &ipmmu_ops, &pdev->dev); in ipmmu_probe()
1110 iommu_device_sysfs_remove(&mmu->iommu); in ipmmu_remove()
1111 iommu_device_unregister(&mmu->iommu); in ipmmu_remove()
1113 arm_iommu_release_mapping(mmu->mapping); in ipmmu_remove()
1130 for (i = 0; i < mmu->num_ctx; i++) { in ipmmu_resume_noirq()
1131 if (!mmu->domains[i]) in ipmmu_resume_noirq()
1134 ipmmu_domain_setup_context(mmu->domains[i]); in ipmmu_resume_noirq()
1138 /* Re-enable active micro-TLBs */ in ipmmu_resume_noirq()
1139 for (i = 0; i < mmu->features->num_utlbs; i++) { in ipmmu_resume_noirq()
1140 if (mmu->utlb_ctx[i] == IPMMU_CTX_INVALID) in ipmmu_resume_noirq()
1143 ipmmu_utlb_enable(mmu->root->domains[mmu->utlb_ctx[i]], i); in ipmmu_resume_noirq()
1159 .name = "ipmmu-vmsa",