3  * IOMMU API for ARM architected SMMU implementations.
13  *	- Non-secure access to the SMMU
18 #define pr_fmt(fmt) "arm-smmu: " fmt
44 #include "arm-smmu.h"
47  * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
61 …"Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' f…
66 …domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
74 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)  in arm_smmu_rpm_get()  argument
76 	if (pm_runtime_enabled(smmu->dev))  in arm_smmu_rpm_get()
77 		return pm_runtime_get_sync(smmu->dev);  in arm_smmu_rpm_get()
82 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)  in arm_smmu_rpm_put()  argument
84 	if (pm_runtime_enabled(smmu->dev))  in arm_smmu_rpm_put()
85 		pm_runtime_put_autosuspend(smmu->dev);  in arm_smmu_rpm_put()
135 					   struct arm_smmu_device **smmu)  in arm_smmu_register_legacy_master()  argument
178 	*smmu = dev_get_drvdata(smmu_dev);  in arm_smmu_register_legacy_master()
188  * delay setting bus ops until we're sure every possible SMMU is ready,
200 					   struct arm_smmu_device **smmu)  in arm_smmu_register_legacy_master()  argument
212 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,  in __arm_smmu_tlb_sync()  argument
218 	if (smmu->impl && unlikely(smmu->impl->tlb_sync))  in __arm_smmu_tlb_sync()
219 		return smmu->impl->tlb_sync(smmu, page, sync, status);  in __arm_smmu_tlb_sync()
221 	arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);  in __arm_smmu_tlb_sync()
224 			reg = arm_smmu_readl(smmu, page, status);  in __arm_smmu_tlb_sync()
231 	dev_err_ratelimited(smmu->dev,  in __arm_smmu_tlb_sync()
232 			    "TLB sync timed out -- SMMU may be deadlocked\n");  in __arm_smmu_tlb_sync()
235 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)  in arm_smmu_tlb_sync_global()  argument
239 	spin_lock_irqsave(&smmu->global_sync_lock, flags);  in arm_smmu_tlb_sync_global()
240 	__arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,  in arm_smmu_tlb_sync_global()
242 	spin_unlock_irqrestore(&smmu->global_sync_lock, flags);  in arm_smmu_tlb_sync_global()
247 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_sync_context()  local
251 	__arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),  in arm_smmu_tlb_sync_context()
264 	arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,  in arm_smmu_tlb_inv_context_s1()
272 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_context_s2()  local
276 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);  in arm_smmu_tlb_inv_context_s2()
277 	arm_smmu_tlb_sync_global(smmu);  in arm_smmu_tlb_inv_context_s2()
284 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_s1()  local
288 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)  in arm_smmu_tlb_inv_range_s1()
295 			arm_smmu_cb_write(smmu, idx, reg, iova);  in arm_smmu_tlb_inv_range_s1()
302 			arm_smmu_cb_writeq(smmu, idx, reg, iova);  in arm_smmu_tlb_inv_range_s1()
312 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_inv_range_s2()  local
315 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)  in arm_smmu_tlb_inv_range_s2()
321 			arm_smmu_cb_writeq(smmu, idx, reg, iova);  in arm_smmu_tlb_inv_range_s2()
323 			arm_smmu_cb_write(smmu, idx, reg, iova);  in arm_smmu_tlb_inv_range_s2()
393 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_tlb_add_page_s2_v1()  local
395 	if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)  in arm_smmu_tlb_add_page_s2_v1()
398 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);  in arm_smmu_tlb_add_page_s2_v1()
428 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_context_fault()  local
431 	fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);  in arm_smmu_context_fault()
435 	fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);  in arm_smmu_context_fault()
436 	iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);  in arm_smmu_context_fault()
437 	cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));  in arm_smmu_context_fault()
439 	dev_err_ratelimited(smmu->dev,  in arm_smmu_context_fault()
443 	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);  in arm_smmu_context_fault()
450 	struct arm_smmu_device *smmu = dev;  in arm_smmu_global_fault()  local
454 	gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);  in arm_smmu_global_fault()
455 	gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);  in arm_smmu_global_fault()
456 	gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);  in arm_smmu_global_fault()
457 	gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);  in arm_smmu_global_fault()
465 			dev_err(smmu->dev,  in arm_smmu_global_fault()
466 …"Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may h…  in arm_smmu_global_fault()
469 			dev_err(smmu->dev,  in arm_smmu_global_fault()
471 		dev_err(smmu->dev,  in arm_smmu_global_fault()
476 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);  in arm_smmu_global_fault()
484 	struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];  in arm_smmu_init_context_bank()
537 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)  in arm_smmu_write_context_bank()  argument
541 	struct arm_smmu_cb *cb = &smmu->cbs[idx];  in arm_smmu_write_context_bank()
546 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);  in arm_smmu_write_context_bank()
553 	if (smmu->version > ARM_SMMU_V1) {  in arm_smmu_write_context_bank()
559 		if (smmu->features & ARM_SMMU_FEAT_VMID16)  in arm_smmu_write_context_bank()
562 		arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);  in arm_smmu_write_context_bank()
567 	if (smmu->version < ARM_SMMU_V2)  in arm_smmu_write_context_bank()
579 	} else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {  in arm_smmu_write_context_bank()
583 	arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);  in arm_smmu_write_context_bank()
590 	if (stage1 && smmu->version > ARM_SMMU_V1)  in arm_smmu_write_context_bank()
591 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);  in arm_smmu_write_context_bank()
592 	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);  in arm_smmu_write_context_bank()
596 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);  in arm_smmu_write_context_bank()
597 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);  in arm_smmu_write_context_bank()
598 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);  in arm_smmu_write_context_bank()
600 		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);  in arm_smmu_write_context_bank()
602 			arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,  in arm_smmu_write_context_bank()
608 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);  in arm_smmu_write_context_bank()
609 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);  in arm_smmu_write_context_bank()
620 	arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);  in arm_smmu_write_context_bank()
624 				       struct arm_smmu_device *smmu,  in arm_smmu_alloc_context_bank()  argument
627 	if (smmu->impl && smmu->impl->alloc_context_bank)  in arm_smmu_alloc_context_bank()
628 		return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);  in arm_smmu_alloc_context_bank()
630 	return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);  in arm_smmu_alloc_context_bank()
634 					struct arm_smmu_device *smmu,  in arm_smmu_init_domain_context()  argument
647 	if (smmu_domain->smmu)  in arm_smmu_init_domain_context()
652 		smmu_domain->smmu = smmu;  in arm_smmu_init_domain_context()
674 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))  in arm_smmu_init_domain_context()
676 	if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))  in arm_smmu_init_domain_context()
687 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)  in arm_smmu_init_domain_context()
691 	    (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&  in arm_smmu_init_domain_context()
695 	    (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |  in arm_smmu_init_domain_context()
708 		start = smmu->num_s2_context_banks;  in arm_smmu_init_domain_context()
709 		ias = smmu->va_size;  in arm_smmu_init_domain_context()
710 		oas = smmu->ipa_size;  in arm_smmu_init_domain_context()
732 		ias = smmu->ipa_size;  in arm_smmu_init_domain_context()
733 		oas = smmu->pa_size;  in arm_smmu_init_domain_context()
741 		if (smmu->version == ARM_SMMU_V2)  in arm_smmu_init_domain_context()
751 	ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);  in arm_smmu_init_domain_context()
756 	smmu_domain->smmu = smmu;  in arm_smmu_init_domain_context()
759 	if (smmu->version < ARM_SMMU_V2) {  in arm_smmu_init_domain_context()
760 		cfg->irptndx = atomic_inc_return(&smmu->irptndx);  in arm_smmu_init_domain_context()
761 		cfg->irptndx %= smmu->num_context_irqs;  in arm_smmu_init_domain_context()
772 		.pgsize_bitmap	= smmu->pgsize_bitmap,  in arm_smmu_init_domain_context()
775 		.coherent_walk	= smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,  in arm_smmu_init_domain_context()
777 		.iommu_dev	= smmu->dev,  in arm_smmu_init_domain_context()
780 	if (smmu->impl && smmu->impl->init_context) {  in arm_smmu_init_domain_context()
781 		ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);  in arm_smmu_init_domain_context()
809 	arm_smmu_write_context_bank(smmu, cfg->cbndx);  in arm_smmu_init_domain_context()
815 	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_init_domain_context()
817 	if (smmu->impl && smmu->impl->context_fault)  in arm_smmu_init_domain_context()
818 		context_fault = smmu->impl->context_fault;  in arm_smmu_init_domain_context()
822 	ret = devm_request_irq(smmu->dev, irq, context_fault,  in arm_smmu_init_domain_context()
823 			       IRQF_SHARED, "arm-smmu-context-fault", domain);  in arm_smmu_init_domain_context()
825 		dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",  in arm_smmu_init_domain_context()
837 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);  in arm_smmu_init_domain_context()
838 	smmu_domain->smmu = NULL;  in arm_smmu_init_domain_context()
847 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_destroy_domain_context()  local
851 	if (!smmu || domain->type == IOMMU_DOMAIN_IDENTITY)  in arm_smmu_destroy_domain_context()
854 	ret = arm_smmu_rpm_get(smmu);  in arm_smmu_destroy_domain_context()
862 	smmu->cbs[cfg->cbndx].cfg = NULL;  in arm_smmu_destroy_domain_context()
863 	arm_smmu_write_context_bank(smmu, cfg->cbndx);  in arm_smmu_destroy_domain_context()
866 		irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];  in arm_smmu_destroy_domain_context()
867 		devm_free_irq(smmu->dev, irq, domain);  in arm_smmu_destroy_domain_context()
871 	__arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);  in arm_smmu_destroy_domain_context()
873 	arm_smmu_rpm_put(smmu);  in arm_smmu_destroy_domain_context()
918 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)  in arm_smmu_write_smr()  argument
920 	struct arm_smmu_smr *smr = smmu->smrs + idx;  in arm_smmu_write_smr()
924 	if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)  in arm_smmu_write_smr()
926 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);  in arm_smmu_write_smr()
929 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)  in arm_smmu_write_s2cr()  argument
931 	struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;  in arm_smmu_write_s2cr()
936 	if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&  in arm_smmu_write_s2cr()
937 	    smmu->smrs[idx].valid)  in arm_smmu_write_s2cr()
939 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);  in arm_smmu_write_s2cr()
942 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)  in arm_smmu_write_sme()  argument
944 	arm_smmu_write_s2cr(smmu, idx);  in arm_smmu_write_sme()
945 	if (smmu->smrs)  in arm_smmu_write_sme()
946 		arm_smmu_write_smr(smmu, idx);  in arm_smmu_write_sme()
953 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)  in arm_smmu_test_smr_masks()  argument
958 	if (!smmu->smrs)  in arm_smmu_test_smr_masks()
968 	for (i = 0; i < smmu->num_mapping_groups; i++)  in arm_smmu_test_smr_masks()
969 		if (!smmu->smrs[i].valid)  in arm_smmu_test_smr_masks()
978 	smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);  in arm_smmu_test_smr_masks()
979 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);  in arm_smmu_test_smr_masks()
980 	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));  in arm_smmu_test_smr_masks()
981 	smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);  in arm_smmu_test_smr_masks()
983 	smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);  in arm_smmu_test_smr_masks()
984 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);  in arm_smmu_test_smr_masks()
985 	smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));  in arm_smmu_test_smr_masks()
986 	smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);  in arm_smmu_test_smr_masks()
989 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)  in arm_smmu_find_sme()  argument
991 	struct arm_smmu_smr *smrs = smmu->smrs;  in arm_smmu_find_sme()
999 	for (i = 0; i < smmu->num_mapping_groups; ++i) {  in arm_smmu_find_sme()
1031 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)  in arm_smmu_free_sme()  argument
1033 	if (--smmu->s2crs[idx].count)  in arm_smmu_free_sme()
1036 	smmu->s2crs[idx] = s2cr_init_val;  in arm_smmu_free_sme()
1037 	if (smmu->smrs)  in arm_smmu_free_sme()
1038 		smmu->smrs[idx].valid = false;  in arm_smmu_free_sme()
1047 	struct arm_smmu_device *smmu = cfg->smmu;  in arm_smmu_master_alloc_smes()  local
1048 	struct arm_smmu_smr *smrs = smmu->smrs;  in arm_smmu_master_alloc_smes()
1051 	mutex_lock(&smmu->stream_map_mutex);  in arm_smmu_master_alloc_smes()
1062 		ret = arm_smmu_find_sme(smmu, sid, mask);  in arm_smmu_master_alloc_smes()
1067 		if (smrs && smmu->s2crs[idx].count == 0) {  in arm_smmu_master_alloc_smes()
1072 		smmu->s2crs[idx].count++;  in arm_smmu_master_alloc_smes()
1078 		arm_smmu_write_sme(smmu, idx);  in arm_smmu_master_alloc_smes()
1080 	mutex_unlock(&smmu->stream_map_mutex);  in arm_smmu_master_alloc_smes()
1085 		arm_smmu_free_sme(smmu, cfg->smendx[i]);  in arm_smmu_master_alloc_smes()
1088 	mutex_unlock(&smmu->stream_map_mutex);  in arm_smmu_master_alloc_smes()
1095 	struct arm_smmu_device *smmu = cfg->smmu;  in arm_smmu_master_free_smes()  local
1098 	mutex_lock(&smmu->stream_map_mutex);  in arm_smmu_master_free_smes()
1100 		if (arm_smmu_free_sme(smmu, idx))  in arm_smmu_master_free_smes()
1101 			arm_smmu_write_sme(smmu, idx);  in arm_smmu_master_free_smes()
1104 	mutex_unlock(&smmu->stream_map_mutex);  in arm_smmu_master_free_smes()
1111 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_domain_add_master()  local
1112 	struct arm_smmu_s2cr *s2cr = smmu->s2crs;  in arm_smmu_domain_add_master()
1129 		arm_smmu_write_s2cr(smmu, idx);  in arm_smmu_domain_add_master()
1139 	struct arm_smmu_device *smmu;  in arm_smmu_attach_dev()  local
1143 		dev_err(dev, "cannot attach to SMMU, is it on the same bus?\n");  in arm_smmu_attach_dev()
1158 	smmu = cfg->smmu;  in arm_smmu_attach_dev()
1160 	ret = arm_smmu_rpm_get(smmu);  in arm_smmu_attach_dev()
1165 	ret = arm_smmu_init_domain_context(domain, smmu, dev);  in arm_smmu_attach_dev()
1173 	if (smmu_domain->smmu != smmu) {  in arm_smmu_attach_dev()
1175 			"cannot attach to SMMU %s whilst already attached to domain on SMMU %s\n",  in arm_smmu_attach_dev()
1176 			dev_name(smmu_domain->smmu->dev), dev_name(smmu->dev));  in arm_smmu_attach_dev()
1195 	pm_runtime_set_autosuspend_delay(smmu->dev, 20);  in arm_smmu_attach_dev()
1196 	pm_runtime_use_autosuspend(smmu->dev);  in arm_smmu_attach_dev()
1199 	arm_smmu_rpm_put(smmu);  in arm_smmu_attach_dev()
1207 	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;  in arm_smmu_map()  local
1213 	arm_smmu_rpm_get(smmu);  in arm_smmu_map()
1215 	arm_smmu_rpm_put(smmu);  in arm_smmu_map()
1224 	struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;  in arm_smmu_unmap()  local
1230 	arm_smmu_rpm_get(smmu);  in arm_smmu_unmap()
1232 	arm_smmu_rpm_put(smmu);  in arm_smmu_unmap()
1240 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_flush_iotlb_all()  local
1243 		arm_smmu_rpm_get(smmu);  in arm_smmu_flush_iotlb_all()
1245 		arm_smmu_rpm_put(smmu);  in arm_smmu_flush_iotlb_all()
1253 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_iotlb_sync()  local
1255 	if (!smmu)  in arm_smmu_iotlb_sync()
1258 	arm_smmu_rpm_get(smmu);  in arm_smmu_iotlb_sync()
1259 	if (smmu->version == ARM_SMMU_V2 ||  in arm_smmu_iotlb_sync()
1263 		arm_smmu_tlb_sync_global(smmu);  in arm_smmu_iotlb_sync()
1264 	arm_smmu_rpm_put(smmu);  in arm_smmu_iotlb_sync()
1271 	struct arm_smmu_device *smmu = smmu_domain->smmu;  in arm_smmu_iova_to_phys_hard()  local
1274 	struct device *dev = smmu->dev;  in arm_smmu_iova_to_phys_hard()
1281 	ret = arm_smmu_rpm_get(smmu);  in arm_smmu_iova_to_phys_hard()
1288 		arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);  in arm_smmu_iova_to_phys_hard()
1290 		arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);  in arm_smmu_iova_to_phys_hard()
1292 	reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;  in arm_smmu_iova_to_phys_hard()
1302 	phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);  in arm_smmu_iova_to_phys_hard()
1310 	arm_smmu_rpm_put(smmu);  in arm_smmu_iova_to_phys_hard()
1327 	if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&  in arm_smmu_iova_to_phys()
1339 		 * Return true here as the SMMU can always send out coherent  in arm_smmu_capable()
1361 	struct arm_smmu_device *smmu = NULL;  in arm_smmu_probe_device()  local
1367 		ret = arm_smmu_register_legacy_master(dev, &smmu);  in arm_smmu_probe_device()
1378 		smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);  in arm_smmu_probe_device()
1388 		if (sid & ~smmu->streamid_mask) {  in arm_smmu_probe_device()
1389 			dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",  in arm_smmu_probe_device()
1390 				sid, smmu->streamid_mask);  in arm_smmu_probe_device()
1393 		if (mask & ~smmu->smr_mask_mask) {  in arm_smmu_probe_device()
1394 			dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",  in arm_smmu_probe_device()
1395 				mask, smmu->smr_mask_mask);  in arm_smmu_probe_device()
1406 	cfg->smmu = smmu;  in arm_smmu_probe_device()
1411 	ret = arm_smmu_rpm_get(smmu);  in arm_smmu_probe_device()
1416 	arm_smmu_rpm_put(smmu);  in arm_smmu_probe_device()
1421 	device_link_add(dev, smmu->dev,  in arm_smmu_probe_device()
1424 	return &smmu->iommu;  in arm_smmu_probe_device()
1437 	struct arm_smmu_device *smmu;  in arm_smmu_release_device()  local
1444 	smmu = cfg->smmu;  in arm_smmu_release_device()
1446 	ret = arm_smmu_rpm_get(smmu);  in arm_smmu_release_device()
1452 	arm_smmu_rpm_put(smmu);  in arm_smmu_release_device()
1463 	struct arm_smmu_device *smmu = cfg->smmu;  in arm_smmu_device_group()  local
1468 		if (group && smmu->s2crs[idx].group &&  in arm_smmu_device_group()
1469 		    group != smmu->s2crs[idx].group)  in arm_smmu_device_group()
1472 		group = smmu->s2crs[idx].group;  in arm_smmu_device_group()
1488 			smmu->s2crs[idx].group = group;  in arm_smmu_device_group()
1534 			if (smmu_domain->smmu) {  in arm_smmu_domain_set_attr()
1599 	const struct arm_smmu_impl *impl = cfg->smmu->impl;  in arm_smmu_def_domain_type()
1629 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)  in arm_smmu_device_reset()  argument
1635 	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);  in arm_smmu_device_reset()
1636 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);  in arm_smmu_device_reset()
1642 	for (i = 0; i < smmu->num_mapping_groups; ++i)  in arm_smmu_device_reset()
1643 		arm_smmu_write_sme(smmu, i);  in arm_smmu_device_reset()
1646 	for (i = 0; i < smmu->num_context_banks; ++i) {  in arm_smmu_device_reset()
1647 		arm_smmu_write_context_bank(smmu, i);  in arm_smmu_device_reset()
1648 		arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_FSR_FAULT);  in arm_smmu_device_reset()
1652 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);  in arm_smmu_device_reset()
1653 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);  in arm_smmu_device_reset()
1655 	reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);  in arm_smmu_device_reset()
1677 	if (smmu->features & ARM_SMMU_FEAT_VMID16)  in arm_smmu_device_reset()
1680 	if (smmu->features & ARM_SMMU_FEAT_EXIDS)  in arm_smmu_device_reset()
1683 	if (smmu->impl && smmu->impl->reset)  in arm_smmu_device_reset()
1684 		smmu->impl->reset(smmu);  in arm_smmu_device_reset()
1687 	arm_smmu_tlb_sync_global(smmu);  in arm_smmu_device_reset()
1688 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);  in arm_smmu_device_reset()
1710 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)  in arm_smmu_device_cfg_probe()  argument
1714 	bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;  in arm_smmu_device_cfg_probe()
1717 	dev_notice(smmu->dev, "probing hardware configuration...\n");  in arm_smmu_device_cfg_probe()
1718 	dev_notice(smmu->dev, "SMMUv%d with:\n",  in arm_smmu_device_cfg_probe()
1719 			smmu->version == ARM_SMMU_V2 ? 2 : 1);  in arm_smmu_device_cfg_probe()
1722 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);  in arm_smmu_device_cfg_probe()
1731 		smmu->features |= ARM_SMMU_FEAT_TRANS_S1;  in arm_smmu_device_cfg_probe()
1732 		dev_notice(smmu->dev, "\tstage 1 translation\n");  in arm_smmu_device_cfg_probe()
1736 		smmu->features |= ARM_SMMU_FEAT_TRANS_S2;  in arm_smmu_device_cfg_probe()
1737 		dev_notice(smmu->dev, "\tstage 2 translation\n");  in arm_smmu_device_cfg_probe()
1741 		smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;  in arm_smmu_device_cfg_probe()
1742 		dev_notice(smmu->dev, "\tnested translation\n");  in arm_smmu_device_cfg_probe()
1745 	if (!(smmu->features &  in arm_smmu_device_cfg_probe()
1747 		dev_err(smmu->dev, "\tno translation support!\n");  in arm_smmu_device_cfg_probe()
1752 	    ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {  in arm_smmu_device_cfg_probe()
1753 		smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;  in arm_smmu_device_cfg_probe()
1754 		dev_notice(smmu->dev, "\taddress translation ops\n");  in arm_smmu_device_cfg_probe()
1765 		dev_notice(smmu->dev, "\t%scoherent table walk\n",  in arm_smmu_device_cfg_probe()
1768 		dev_notice(smmu->dev,  in arm_smmu_device_cfg_probe()
1772 	if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {  in arm_smmu_device_cfg_probe()
1773 		smmu->features |= ARM_SMMU_FEAT_EXIDS;  in arm_smmu_device_cfg_probe()
1778 	smmu->streamid_mask = size - 1;  in arm_smmu_device_cfg_probe()
1780 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;  in arm_smmu_device_cfg_probe()
1783 			dev_err(smmu->dev,  in arm_smmu_device_cfg_probe()
1789 		smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),  in arm_smmu_device_cfg_probe()
1791 		if (!smmu->smrs)  in arm_smmu_device_cfg_probe()
1794 		dev_notice(smmu->dev,  in arm_smmu_device_cfg_probe()
1798 	smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),  in arm_smmu_device_cfg_probe()
1800 	if (!smmu->s2crs)  in arm_smmu_device_cfg_probe()
1803 		smmu->s2crs[i] = s2cr_init_val;  in arm_smmu_device_cfg_probe()
1805 	smmu->num_mapping_groups = size;  in arm_smmu_device_cfg_probe()
1806 	mutex_init(&smmu->stream_map_mutex);  in arm_smmu_device_cfg_probe()
1807 	spin_lock_init(&smmu->global_sync_lock);  in arm_smmu_device_cfg_probe()
1809 	if (smmu->version < ARM_SMMU_V2 ||  in arm_smmu_device_cfg_probe()
1811 		smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;  in arm_smmu_device_cfg_probe()
1813 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;  in arm_smmu_device_cfg_probe()
1817 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);  in arm_smmu_device_cfg_probe()
1818 	smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;  in arm_smmu_device_cfg_probe()
1820 	/* Check for size mismatch of SMMU address space from mapped region */  in arm_smmu_device_cfg_probe()
1822 	if (smmu->numpage != 2 * size << smmu->pgshift)  in arm_smmu_device_cfg_probe()
1823 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1824 			"SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",  in arm_smmu_device_cfg_probe()
1825 			2 * size << smmu->pgshift, smmu->numpage);  in arm_smmu_device_cfg_probe()
1827 	smmu->numpage = size;  in arm_smmu_device_cfg_probe()
1829 	smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);  in arm_smmu_device_cfg_probe()
1830 	smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);  in arm_smmu_device_cfg_probe()
1831 	if (smmu->num_s2_context_banks > smmu->num_context_banks) {  in arm_smmu_device_cfg_probe()
1832 		dev_err(smmu->dev, "impossible number of S2 context banks!\n");  in arm_smmu_device_cfg_probe()
1835 	dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",  in arm_smmu_device_cfg_probe()
1836 		   smmu->num_context_banks, smmu->num_s2_context_banks);  in arm_smmu_device_cfg_probe()
1837 	smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,  in arm_smmu_device_cfg_probe()
1838 				 sizeof(*smmu->cbs), GFP_KERNEL);  in arm_smmu_device_cfg_probe()
1839 	if (!smmu->cbs)  in arm_smmu_device_cfg_probe()
1843 	id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);  in arm_smmu_device_cfg_probe()
1845 	smmu->ipa_size = size;  in arm_smmu_device_cfg_probe()
1849 	smmu->pa_size = size;  in arm_smmu_device_cfg_probe()
1852 		smmu->features |= ARM_SMMU_FEAT_VMID16;  in arm_smmu_device_cfg_probe()
1859 	if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))  in arm_smmu_device_cfg_probe()
1860 		dev_warn(smmu->dev,  in arm_smmu_device_cfg_probe()
1863 	if (smmu->version < ARM_SMMU_V2) {  in arm_smmu_device_cfg_probe()
1864 		smmu->va_size = smmu->ipa_size;  in arm_smmu_device_cfg_probe()
1865 		if (smmu->version == ARM_SMMU_V1_64K)  in arm_smmu_device_cfg_probe()
1866 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;  in arm_smmu_device_cfg_probe()
1869 		smmu->va_size = arm_smmu_id_size_to_bits(size);  in arm_smmu_device_cfg_probe()
1871 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;  in arm_smmu_device_cfg_probe()
1873 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;  in arm_smmu_device_cfg_probe()
1875 			smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;  in arm_smmu_device_cfg_probe()
1878 	if (smmu->impl && smmu->impl->cfg_probe) {  in arm_smmu_device_cfg_probe()
1879 		ret = smmu->impl->cfg_probe(smmu);  in arm_smmu_device_cfg_probe()
1885 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)  in arm_smmu_device_cfg_probe()
1886 		smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;  in arm_smmu_device_cfg_probe()
1887 	if (smmu->features &  in arm_smmu_device_cfg_probe()
1889 		smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;  in arm_smmu_device_cfg_probe()
1890 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)  in arm_smmu_device_cfg_probe()
1891 		smmu->pgsize_bitmap |= SZ_16K | SZ_32M;  in arm_smmu_device_cfg_probe()
1892 	if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)  in arm_smmu_device_cfg_probe()
1893 		smmu->pgsize_bitmap |= SZ_64K | SZ_512M;  in arm_smmu_device_cfg_probe()
1896 		arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;  in arm_smmu_device_cfg_probe()
1898 		arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;  in arm_smmu_device_cfg_probe()
1899 	dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",  in arm_smmu_device_cfg_probe()
1900 		   smmu->pgsize_bitmap);  in arm_smmu_device_cfg_probe()
1903 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)  in arm_smmu_device_cfg_probe()
1904 		dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",  in arm_smmu_device_cfg_probe()
1905 			   smmu->va_size, smmu->ipa_size);  in arm_smmu_device_cfg_probe()
1907 	if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)  in arm_smmu_device_cfg_probe()
1908 		dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",  in arm_smmu_device_cfg_probe()
1909 			   smmu->ipa_size, smmu->pa_size);  in arm_smmu_device_cfg_probe()
1930 	{ .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1931 	{ .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1935 	{ .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1936 	{ .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1937 	{ .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1943 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)  in acpi_smmu_get_data()  argument
1950 		smmu->version = ARM_SMMU_V1;  in acpi_smmu_get_data()
1951 		smmu->model = GENERIC_SMMU;  in acpi_smmu_get_data()
1954 		smmu->version = ARM_SMMU_V1_64K;  in acpi_smmu_get_data()
1955 		smmu->model = GENERIC_SMMU;  in acpi_smmu_get_data()
1958 		smmu->version = ARM_SMMU_V2;  in acpi_smmu_get_data()
1959 		smmu->model = GENERIC_SMMU;  in acpi_smmu_get_data()
1962 		smmu->version = ARM_SMMU_V2;  in acpi_smmu_get_data()
1963 		smmu->model = ARM_MMU500;  in acpi_smmu_get_data()
1966 		smmu->version = ARM_SMMU_V2;  in acpi_smmu_get_data()
1967 		smmu->model = CAVIUM_SMMUV2;  in acpi_smmu_get_data()
1977 				      struct arm_smmu_device *smmu)  in arm_smmu_device_acpi_probe()  argument
1979 	struct device *dev = smmu->dev;  in arm_smmu_device_acpi_probe()
1988 	ret = acpi_smmu_get_data(iort_smmu->model, smmu);  in arm_smmu_device_acpi_probe()
1993 	smmu->num_global_irqs = 1;  in arm_smmu_device_acpi_probe()
1996 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;  in arm_smmu_device_acpi_probe()
2002 					     struct arm_smmu_device *smmu)  in arm_smmu_device_acpi_probe()  argument
2009 				    struct arm_smmu_device *smmu)  in arm_smmu_device_dt_probe()  argument
2016 				 &smmu->num_global_irqs)) {  in arm_smmu_device_dt_probe()
2022 	smmu->version = data->version;  in arm_smmu_device_dt_probe()
2023 	smmu->model = data->model;  in arm_smmu_device_dt_probe()
2029 				  IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");  in arm_smmu_device_dt_probe()
2040 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;  in arm_smmu_device_dt_probe()
2095 	struct arm_smmu_device *smmu;  in arm_smmu_device_probe()  local
2100 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in arm_smmu_device_probe()
2101 	if (!smmu) {  in arm_smmu_device_probe()
2105 	smmu->dev = dev;  in arm_smmu_device_probe()
2108 		err = arm_smmu_device_dt_probe(pdev, smmu);  in arm_smmu_device_probe()
2110 		err = arm_smmu_device_acpi_probe(pdev, smmu);  in arm_smmu_device_probe()
2117 	smmu->base = devm_ioremap_resource(dev, res);  in arm_smmu_device_probe()
2118 	if (IS_ERR(smmu->base))  in arm_smmu_device_probe()
2119 		return PTR_ERR(smmu->base);  in arm_smmu_device_probe()
2124 	smmu->numpage = resource_size(res);  in arm_smmu_device_probe()
2126 	smmu = arm_smmu_impl_init(smmu);  in arm_smmu_device_probe()
2127 	if (IS_ERR(smmu))  in arm_smmu_device_probe()
2128 		return PTR_ERR(smmu);  in arm_smmu_device_probe()
2133 		if (num_irqs > smmu->num_global_irqs)  in arm_smmu_device_probe()
2134 			smmu->num_context_irqs++;  in arm_smmu_device_probe()
2137 	if (!smmu->num_context_irqs) {  in arm_smmu_device_probe()
2139 			num_irqs, smmu->num_global_irqs + 1);  in arm_smmu_device_probe()
2143 	smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),  in arm_smmu_device_probe()
2145 	if (!smmu->irqs) {  in arm_smmu_device_probe()
2155 		smmu->irqs[i] = irq;  in arm_smmu_device_probe()
2158 	err = devm_clk_bulk_get_all(dev, &smmu->clks);  in arm_smmu_device_probe()
2163 	smmu->num_clks = err;  in arm_smmu_device_probe()
2165 	err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);  in arm_smmu_device_probe()
2169 	err = arm_smmu_device_cfg_probe(smmu);  in arm_smmu_device_probe()
2173 	if (smmu->version == ARM_SMMU_V2) {  in arm_smmu_device_probe()
2174 		if (smmu->num_context_banks > smmu->num_context_irqs) {  in arm_smmu_device_probe()
2177 			      smmu->num_context_irqs, smmu->num_context_banks);  in arm_smmu_device_probe()
2182 		smmu->num_context_irqs = smmu->num_context_banks;  in arm_smmu_device_probe()
2185 	if (smmu->impl && smmu->impl->global_fault)  in arm_smmu_device_probe()
2186 		global_fault = smmu->impl->global_fault;  in arm_smmu_device_probe()
2190 	for (i = 0; i < smmu->num_global_irqs; ++i) {  in arm_smmu_device_probe()
2191 		err = devm_request_irq(smmu->dev, smmu->irqs[i],  in arm_smmu_device_probe()
2194 				       "arm-smmu global fault",  in arm_smmu_device_probe()
2195 				       smmu);  in arm_smmu_device_probe()
2198 				i, smmu->irqs[i]);  in arm_smmu_device_probe()
2203 	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,  in arm_smmu_device_probe()
2204 				     "smmu.%pa", &ioaddr);  in arm_smmu_device_probe()
2210 	iommu_device_set_ops(&smmu->iommu, &arm_smmu_ops);  in arm_smmu_device_probe()
2211 	iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);  in arm_smmu_device_probe()
2213 	err = iommu_device_register(&smmu->iommu);  in arm_smmu_device_probe()
2219 	platform_set_drvdata(pdev, smmu);  in arm_smmu_device_probe()
2220 	arm_smmu_device_reset(smmu);  in arm_smmu_device_probe()
2221 	arm_smmu_test_smr_masks(smmu);  in arm_smmu_device_probe()
2235 	 * For ACPI and generic DT bindings, an SMMU will be probed before  in arm_smmu_device_probe()
2237 	 * ready to handle default domain setup as soon as any SMMU exists.  in arm_smmu_device_probe()
2247 	struct arm_smmu_device *smmu = platform_get_drvdata(pdev);  in arm_smmu_device_remove()  local
2249 	if (!smmu)  in arm_smmu_device_remove()
2252 	if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))  in arm_smmu_device_remove()
2256 	iommu_device_unregister(&smmu->iommu);  in arm_smmu_device_remove()
2257 	iommu_device_sysfs_remove(&smmu->iommu);  in arm_smmu_device_remove()
2259 	arm_smmu_rpm_get(smmu);  in arm_smmu_device_remove()
2261 	arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);  in arm_smmu_device_remove()
2262 	arm_smmu_rpm_put(smmu);  in arm_smmu_device_remove()
2264 	if (pm_runtime_enabled(smmu->dev))  in arm_smmu_device_remove()
2265 		pm_runtime_force_suspend(smmu->dev);  in arm_smmu_device_remove()
2267 		clk_bulk_disable(smmu->num_clks, smmu->clks);  in arm_smmu_device_remove()
2269 	clk_bulk_unprepare(smmu->num_clks, smmu->clks);  in arm_smmu_device_remove()
2280 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);  in arm_smmu_runtime_resume()  local
2283 	ret = clk_bulk_enable(smmu->num_clks, smmu->clks);  in arm_smmu_runtime_resume()
2287 	arm_smmu_device_reset(smmu);  in arm_smmu_runtime_resume()
2294 	struct arm_smmu_device *smmu = dev_get_drvdata(dev);  in arm_smmu_runtime_suspend()  local
2296 	clk_bulk_disable(smmu->num_clks, smmu->clks);  in arm_smmu_runtime_suspend()
2325 		.name			= "arm-smmu",
2336 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2338 MODULE_ALIAS("platform:arm-smmu");