23 	struct tegra_smmu *smmu;  member
53 	struct tegra_smmu *smmu;  member
69 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,  in smmu_writel()  argument
72 	writel(value, smmu->regs + offset);  in smmu_writel()
75 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)  in smmu_readl()  argument
77 	return readl(smmu->regs + offset);  in smmu_readl()
86 #define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \  argument
87 	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
165 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)  in smmu_dma_addr_valid()  argument
168 	return (addr & smmu->pfn_mask) == addr;  in smmu_dma_addr_valid()
171 static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde)  in smmu_pde_to_dma()  argument
173 	return (dma_addr_t)(pde & smmu->pfn_mask) << 12;  in smmu_pde_to_dma()
176 static void smmu_flush_ptc_all(struct tegra_smmu *smmu)  in smmu_flush_ptc_all()  argument
178 	smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);  in smmu_flush_ptc_all()
181 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,  in smmu_flush_ptc()  argument
186 	offset &= ~(smmu->mc->soc->atom_size - 1);  in smmu_flush_ptc()
188 	if (smmu->mc->soc->num_address_bits > 32) {  in smmu_flush_ptc()
194 		smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);  in smmu_flush_ptc()
198 	smmu_writel(smmu, value, SMMU_PTC_FLUSH);  in smmu_flush_ptc()
201 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)  in smmu_flush_tlb()  argument
203 	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);  in smmu_flush_tlb()
206 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,  in smmu_flush_tlb_asid()  argument
211 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_asid()
217 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_asid()
220 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,  in smmu_flush_tlb_section()  argument
226 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_section()
232 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_section()
235 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,  in smmu_flush_tlb_group()  argument
241 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_group()
247 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_group()
250 static inline void smmu_flush(struct tegra_smmu *smmu)  in smmu_flush()  argument
252 	smmu_readl(smmu, SMMU_PTB_ASID);  in smmu_flush()
255 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)  in tegra_smmu_alloc_asid()  argument
259 	mutex_lock(&smmu->lock);  in tegra_smmu_alloc_asid()
261 	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);  in tegra_smmu_alloc_asid()
262 	if (id >= smmu->soc->num_asids) {  in tegra_smmu_alloc_asid()
263 		mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
267 	set_bit(id, smmu->asids);  in tegra_smmu_alloc_asid()
270 	mutex_unlock(&smmu->lock);  in tegra_smmu_alloc_asid()
274 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)  in tegra_smmu_free_asid()  argument
276 	mutex_lock(&smmu->lock);  in tegra_smmu_free_asid()
277 	clear_bit(id, smmu->asids);  in tegra_smmu_free_asid()
278 	mutex_unlock(&smmu->lock);  in tegra_smmu_free_asid()
343 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_swgroup()  argument
348 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_find_swgroup()
349 		if (smmu->soc->swgroups[i].swgroup == swgroup) {  in tegra_smmu_find_swgroup()
350 			group = &smmu->soc->swgroups[i];  in tegra_smmu_find_swgroup()
358 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_enable()  argument
365 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_enable()
367 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_enable()
371 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_enable()
379 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_enable()
380 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_enable()
385 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_enable()
386 		value |= BIT(client->smmu.bit);  in tegra_smmu_enable()
387 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_enable()
391 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_disable()  argument
398 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_disable()
400 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_disable()
404 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_disable()
407 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_disable()
408 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_disable()
413 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_disable()
414 		value &= ~BIT(client->smmu.bit);  in tegra_smmu_disable()
415 		smmu_writel(smmu, value, client->smmu.reg);  in tegra_smmu_disable()
419 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,  in tegra_smmu_as_prepare()  argument
430 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
432 	if (dma_mapping_error(smmu->dev, as->pd_dma))  in tegra_smmu_as_prepare()
436 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
441 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
445 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
446 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
448 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
450 	smmu_writel(smmu, value, SMMU_PTB_DATA);  in tegra_smmu_as_prepare()
451 	smmu_flush(smmu);  in tegra_smmu_as_prepare()
453 	as->smmu = smmu;  in tegra_smmu_as_prepare()
459 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
463 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,  in tegra_smmu_as_unprepare()  argument
469 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
471 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
473 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
479 	struct tegra_smmu *smmu = dev_iommu_priv_get(dev);  in tegra_smmu_attach_dev()  local
490 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_attach_dev()
497 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
501 		tegra_smmu_enable(smmu, swgroup, as->id);  in tegra_smmu_attach_dev()
515 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()  local
523 		if (args.np != smmu->dev->of_node) {  in tegra_smmu_detach_dev()
530 		tegra_smmu_disable(smmu, swgroup, as->id);  in tegra_smmu_detach_dev()
531 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_detach_dev()
540 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()  local
548 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
552 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
553 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
554 	smmu_flush(smmu);  in tegra_smmu_set_pde()
568 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_lookup()  local
577 	*dmap = smmu_pde_to_dma(smmu, pd[pd_index]);  in tegra_smmu_pte_lookup()
586 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()  local
591 		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,  in as_get_pte()
593 		if (dma_mapping_error(smmu->dev, dma)) {  in as_get_pte()
598 		if (!smmu_dma_addr_valid(smmu, dma)) {  in as_get_pte()
599 			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,  in as_get_pte()
614 		*dmap = smmu_pde_to_dma(smmu, pd[pde]);  in as_get_pte()
637 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()  local
639 		dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]);  in tegra_smmu_pte_put_use()
643 		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);  in tegra_smmu_pte_put_use()
652 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()  local
657 	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,  in tegra_smmu_set_pte()
659 	smmu_flush_ptc(smmu, pte_dma, offset);  in tegra_smmu_set_pte()
660 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
661 	smmu_flush(smmu);  in tegra_smmu_set_pte()
797 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
815 	return mc->smmu;  in tegra_smmu_find()
818 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,  in tegra_smmu_configure()  argument
821 	const struct iommu_ops *ops = smmu->iommu.ops;  in tegra_smmu_configure()
843 	struct tegra_smmu *smmu = NULL;  in tegra_smmu_probe_device()  local
850 		smmu = tegra_smmu_find(args.np);  in tegra_smmu_probe_device()
851 		if (smmu) {  in tegra_smmu_probe_device()
852 			err = tegra_smmu_configure(smmu, dev, &args);  in tegra_smmu_probe_device()
863 			dev_iommu_priv_set(dev, smmu);  in tegra_smmu_probe_device()
872 	if (!smmu)  in tegra_smmu_probe_device()
875 	return &smmu->iommu;  in tegra_smmu_probe_device()
884 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_group()  argument
888 	for (i = 0; i < smmu->soc->num_groups; i++)  in tegra_smmu_find_group()
889 		for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)  in tegra_smmu_find_group()
890 			if (smmu->soc->groups[i].swgroups[j] == swgroup)  in tegra_smmu_find_group()
891 				return &smmu->soc->groups[i];  in tegra_smmu_find_group()
899 	struct tegra_smmu *smmu = group->smmu;  in tegra_smmu_group_release()  local
901 	mutex_lock(&smmu->lock);  in tegra_smmu_group_release()
903 	mutex_unlock(&smmu->lock);  in tegra_smmu_group_release()
906 static struct iommu_group *tegra_smmu_group_get(struct tegra_smmu *smmu,  in tegra_smmu_group_get()  argument
914 	soc = tegra_smmu_find_group(smmu, swgroup);  in tegra_smmu_group_get()
916 	mutex_lock(&smmu->lock);  in tegra_smmu_group_get()
919 	list_for_each_entry(group, &smmu->groups, list)  in tegra_smmu_group_get()
922 			mutex_unlock(&smmu->lock);  in tegra_smmu_group_get()
926 	group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);  in tegra_smmu_group_get()
928 		mutex_unlock(&smmu->lock);  in tegra_smmu_group_get()
934 	group->smmu = smmu;  in tegra_smmu_group_get()
939 		devm_kfree(smmu->dev, group);  in tegra_smmu_group_get()
940 		mutex_unlock(&smmu->lock);  in tegra_smmu_group_get()
947 	list_add_tail(&group->list, &smmu->groups);  in tegra_smmu_group_get()
948 	mutex_unlock(&smmu->lock);  in tegra_smmu_group_get()
956 	struct tegra_smmu *smmu = dev_iommu_priv_get(dev);  in tegra_smmu_device_group()  local
959 	group = tegra_smmu_group_get(smmu, fwspec->ids[0]);  in tegra_smmu_device_group()
1007 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_swgroups_show()  local
1014 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_swgroups_show()
1015 		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];  in tegra_smmu_swgroups_show()
1019 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_swgroups_show()
1039 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_clients_show()  local
1046 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_clients_show()
1047 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_clients_show()
1050 		value = smmu_readl(smmu, client->smmu.reg);  in tegra_smmu_clients_show()
1052 		if (value & BIT(client->smmu.bit))  in tegra_smmu_clients_show()
1065 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_init()  argument
1067 	smmu->debugfs = debugfs_create_dir("smmu", NULL);  in tegra_smmu_debugfs_init()
1068 	if (!smmu->debugfs)  in tegra_smmu_debugfs_init()
1071 	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1073 	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1077 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_exit()  argument
1079 	debugfs_remove_recursive(smmu->debugfs);  in tegra_smmu_debugfs_exit()
1086 	struct tegra_smmu *smmu;  in tegra_smmu_probe()  local
1091 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in tegra_smmu_probe()
1092 	if (!smmu)  in tegra_smmu_probe()
1103 	mc->smmu = smmu;  in tegra_smmu_probe()
1107 	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);  in tegra_smmu_probe()
1108 	if (!smmu->asids)  in tegra_smmu_probe()
1111 	INIT_LIST_HEAD(&smmu->groups);  in tegra_smmu_probe()
1112 	mutex_init(&smmu->lock);  in tegra_smmu_probe()
1114 	smmu->regs = mc->regs;  in tegra_smmu_probe()
1115 	smmu->soc = soc;  in tegra_smmu_probe()
1116 	smmu->dev = dev;  in tegra_smmu_probe()
1117 	smmu->mc = mc;  in tegra_smmu_probe()
1119 	smmu->pfn_mask =  in tegra_smmu_probe()
1122 		mc->soc->num_address_bits, smmu->pfn_mask);  in tegra_smmu_probe()
1123 	smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;  in tegra_smmu_probe()
1124 	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,  in tegra_smmu_probe()
1125 		smmu->tlb_mask);  in tegra_smmu_probe()
1132 	smmu_writel(smmu, value, SMMU_PTC_CONFIG);  in tegra_smmu_probe()
1135 		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);  in tegra_smmu_probe()
1140 	smmu_writel(smmu, value, SMMU_TLB_CONFIG);  in tegra_smmu_probe()
1142 	smmu_flush_ptc_all(smmu);  in tegra_smmu_probe()
1143 	smmu_flush_tlb(smmu);  in tegra_smmu_probe()
1144 	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);  in tegra_smmu_probe()
1145 	smmu_flush(smmu);  in tegra_smmu_probe()
1149 	err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));  in tegra_smmu_probe()
1153 	iommu_device_set_ops(&smmu->iommu, &tegra_smmu_ops);  in tegra_smmu_probe()
1154 	iommu_device_set_fwnode(&smmu->iommu, dev->fwnode);  in tegra_smmu_probe()
1156 	err = iommu_device_register(&smmu->iommu);  in tegra_smmu_probe()
1158 		iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_probe()
1164 		iommu_device_unregister(&smmu->iommu);  in tegra_smmu_probe()
1165 		iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_probe()
1170 		tegra_smmu_debugfs_init(smmu);  in tegra_smmu_probe()
1172 	return smmu;  in tegra_smmu_probe()
1175 void tegra_smmu_remove(struct tegra_smmu *smmu)  in tegra_smmu_remove()  argument
1177 	iommu_device_unregister(&smmu->iommu);  in tegra_smmu_remove()
1178 	iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_remove()
1181 		tegra_smmu_debugfs_exit(smmu);  in tegra_smmu_remove()