24 	struct tegra_smmu *smmu;  member
54 	struct tegra_smmu *smmu;  member
70 static inline void smmu_writel(struct tegra_smmu *smmu, u32 value,  in smmu_writel()  argument
73 	writel(value, smmu->regs + offset);  in smmu_writel()
76 static inline u32 smmu_readl(struct tegra_smmu *smmu, unsigned long offset)  in smmu_readl()  argument
78 	return readl(smmu->regs + offset);  in smmu_readl()
87 #define  SMMU_TLB_CONFIG_ACTIVE_LINES(smmu) \  argument
88 	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
166 static bool smmu_dma_addr_valid(struct tegra_smmu *smmu, dma_addr_t addr)  in smmu_dma_addr_valid()  argument
169 	return (addr & smmu->pfn_mask) == addr;  in smmu_dma_addr_valid()
172 static dma_addr_t smmu_pde_to_dma(struct tegra_smmu *smmu, u32 pde)  in smmu_pde_to_dma()  argument
174 	return (dma_addr_t)(pde & smmu->pfn_mask) << 12;  in smmu_pde_to_dma()
177 static void smmu_flush_ptc_all(struct tegra_smmu *smmu)  in smmu_flush_ptc_all()  argument
179 	smmu_writel(smmu, SMMU_PTC_FLUSH_TYPE_ALL, SMMU_PTC_FLUSH);  in smmu_flush_ptc_all()
182 static inline void smmu_flush_ptc(struct tegra_smmu *smmu, dma_addr_t dma,  in smmu_flush_ptc()  argument
187 	offset &= ~(smmu->mc->soc->atom_size - 1);  in smmu_flush_ptc()
189 	if (smmu->mc->soc->num_address_bits > 32) {  in smmu_flush_ptc()
195 		smmu_writel(smmu, value, SMMU_PTC_FLUSH_HI);  in smmu_flush_ptc()
199 	smmu_writel(smmu, value, SMMU_PTC_FLUSH);  in smmu_flush_ptc()
202 static inline void smmu_flush_tlb(struct tegra_smmu *smmu)  in smmu_flush_tlb()  argument
204 	smmu_writel(smmu, SMMU_TLB_FLUSH_VA_MATCH_ALL, SMMU_TLB_FLUSH);  in smmu_flush_tlb()
207 static inline void smmu_flush_tlb_asid(struct tegra_smmu *smmu,  in smmu_flush_tlb_asid()  argument
212 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_asid()
218 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_asid()
221 static inline void smmu_flush_tlb_section(struct tegra_smmu *smmu,  in smmu_flush_tlb_section()  argument
227 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_section()
233 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_section()
236 static inline void smmu_flush_tlb_group(struct tegra_smmu *smmu,  in smmu_flush_tlb_group()  argument
242 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_group()
248 	smmu_writel(smmu, value, SMMU_TLB_FLUSH);  in smmu_flush_tlb_group()
251 static inline void smmu_flush(struct tegra_smmu *smmu)  in smmu_flush()  argument
253 	smmu_readl(smmu, SMMU_PTB_ASID);  in smmu_flush()
256 static int tegra_smmu_alloc_asid(struct tegra_smmu *smmu, unsigned int *idp)  in tegra_smmu_alloc_asid()  argument
260 	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);  in tegra_smmu_alloc_asid()
261 	if (id >= smmu->soc->num_asids)  in tegra_smmu_alloc_asid()
264 	set_bit(id, smmu->asids);  in tegra_smmu_alloc_asid()
270 static void tegra_smmu_free_asid(struct tegra_smmu *smmu, unsigned int id)  in tegra_smmu_free_asid()  argument
272 	clear_bit(id, smmu->asids);  in tegra_smmu_free_asid()
337 tegra_smmu_find_swgroup(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_swgroup()  argument
342 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_find_swgroup()
343 		if (smmu->soc->swgroups[i].swgroup == swgroup) {  in tegra_smmu_find_swgroup()
344 			group = &smmu->soc->swgroups[i];  in tegra_smmu_find_swgroup()
352 static void tegra_smmu_enable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_enable()  argument
359 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_enable()
361 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_enable()
365 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_enable()
373 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_enable()
374 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_enable()
379 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_enable()
380 		value |= BIT(client->regs.smmu.bit);  in tegra_smmu_enable()
381 		smmu_writel(smmu, value, client->regs.smmu.reg);  in tegra_smmu_enable()
385 static void tegra_smmu_disable(struct tegra_smmu *smmu, unsigned int swgroup,  in tegra_smmu_disable()  argument
392 	group = tegra_smmu_find_swgroup(smmu, swgroup);  in tegra_smmu_disable()
394 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_disable()
398 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_disable()
401 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_disable()
402 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_disable()
407 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_disable()
408 		value &= ~BIT(client->regs.smmu.bit);  in tegra_smmu_disable()
409 		smmu_writel(smmu, value, client->regs.smmu.reg);  in tegra_smmu_disable()
413 static int tegra_smmu_as_prepare(struct tegra_smmu *smmu,  in tegra_smmu_as_prepare()  argument
419 	mutex_lock(&smmu->lock);  in tegra_smmu_as_prepare()
426 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
428 	if (dma_mapping_error(smmu->dev, as->pd_dma)) {  in tegra_smmu_as_prepare()
434 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
439 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
443 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
444 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
446 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
448 	smmu_writel(smmu, value, SMMU_PTB_DATA);  in tegra_smmu_as_prepare()
449 	smmu_flush(smmu);  in tegra_smmu_as_prepare()
451 	as->smmu = smmu;  in tegra_smmu_as_prepare()
454 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_prepare()
459 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
461 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_prepare()
466 static void tegra_smmu_as_unprepare(struct tegra_smmu *smmu,  in tegra_smmu_as_unprepare()  argument
469 	mutex_lock(&smmu->lock);  in tegra_smmu_as_unprepare()
472 		mutex_unlock(&smmu->lock);  in tegra_smmu_as_unprepare()
476 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
478 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
480 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
482 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_unprepare()
489 	struct tegra_smmu *smmu = dev_iommu_priv_get(dev);  in tegra_smmu_attach_dev()  local
498 		err = tegra_smmu_as_prepare(smmu, as);  in tegra_smmu_attach_dev()
502 		tegra_smmu_enable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
512 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
513 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_attach_dev()
523 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()  local
530 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_detach_dev()
531 		tegra_smmu_as_unprepare(smmu, as);  in tegra_smmu_detach_dev()
539 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()  local
547 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
551 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
552 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
553 	smmu_flush(smmu);  in tegra_smmu_set_pde()
567 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_lookup()  local
576 	*dmap = smmu_pde_to_dma(smmu, pd[pd_index]);  in tegra_smmu_pte_lookup()
585 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()  local
590 		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,  in as_get_pte()
592 		if (dma_mapping_error(smmu->dev, dma)) {  in as_get_pte()
597 		if (!smmu_dma_addr_valid(smmu, dma)) {  in as_get_pte()
598 			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,  in as_get_pte()
613 		*dmap = smmu_pde_to_dma(smmu, pd[pde]);  in as_get_pte()
636 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()  local
638 		dma_addr_t pte_dma = smmu_pde_to_dma(smmu, pd[pde]);  in tegra_smmu_pte_put_use()
642 		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);  in tegra_smmu_pte_put_use()
651 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()  local
656 	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,  in tegra_smmu_set_pte()
658 	smmu_flush_ptc(smmu, pte_dma, offset);  in tegra_smmu_set_pte()
659 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
660 	smmu_flush(smmu);  in tegra_smmu_set_pte()
796 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
814 	return mc->smmu;  in tegra_smmu_find()
817 static int tegra_smmu_configure(struct tegra_smmu *smmu, struct device *dev,  in tegra_smmu_configure()  argument
820 	const struct iommu_ops *ops = smmu->iommu.ops;  in tegra_smmu_configure()
842 	struct tegra_smmu *smmu = NULL;  in tegra_smmu_probe_device()  local
849 		smmu = tegra_smmu_find(args.np);  in tegra_smmu_probe_device()
850 		if (smmu) {  in tegra_smmu_probe_device()
851 			err = tegra_smmu_configure(smmu, dev, &args);  in tegra_smmu_probe_device()
863 	smmu = dev_iommu_priv_get(dev);  in tegra_smmu_probe_device()
864 	if (!smmu)  in tegra_smmu_probe_device()
867 	return &smmu->iommu;  in tegra_smmu_probe_device()
873 tegra_smmu_find_group(struct tegra_smmu *smmu, unsigned int swgroup)  in tegra_smmu_find_group()  argument
877 	for (i = 0; i < smmu->soc->num_groups; i++)  in tegra_smmu_find_group()
878 		for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)  in tegra_smmu_find_group()
879 			if (smmu->soc->groups[i].swgroups[j] == swgroup)  in tegra_smmu_find_group()
880 				return &smmu->soc->groups[i];  in tegra_smmu_find_group()
888 	struct tegra_smmu *smmu = group->smmu;  in tegra_smmu_group_release()  local
890 	mutex_lock(&smmu->lock);  in tegra_smmu_group_release()
892 	mutex_unlock(&smmu->lock);  in tegra_smmu_group_release()
898 	struct tegra_smmu *smmu = dev_iommu_priv_get(dev);  in tegra_smmu_device_group()  local
905 	soc = tegra_smmu_find_group(smmu, swgroup);  in tegra_smmu_device_group()
907 	mutex_lock(&smmu->lock);  in tegra_smmu_device_group()
910 	list_for_each_entry(group, &smmu->groups, list)  in tegra_smmu_device_group()
913 			mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
917 	group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);  in tegra_smmu_device_group()
919 		mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
925 	group->smmu = smmu;  in tegra_smmu_device_group()
934 		devm_kfree(smmu->dev, group);  in tegra_smmu_device_group()
935 		mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
942 	list_add_tail(&group->list, &smmu->groups);  in tegra_smmu_device_group()
943 	mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
959 	 * the SMMU parent device is the same as the MC, so the reference count  in tegra_smmu_of_xlate()
964 	dev_iommu_priv_set(dev, mc->smmu);  in tegra_smmu_of_xlate()
1002 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_swgroups_show()  local
1009 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_swgroups_show()
1010 		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];  in tegra_smmu_swgroups_show()
1014 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_swgroups_show()
1034 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_clients_show()  local
1041 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_clients_show()
1042 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_clients_show()
1045 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_clients_show()
1047 		if (value & BIT(client->regs.smmu.bit))  in tegra_smmu_clients_show()
1060 static void tegra_smmu_debugfs_init(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_init()  argument
1062 	smmu->debugfs = debugfs_create_dir("smmu", NULL);  in tegra_smmu_debugfs_init()
1063 	if (!smmu->debugfs)  in tegra_smmu_debugfs_init()
1066 	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1068 	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1072 static void tegra_smmu_debugfs_exit(struct tegra_smmu *smmu)  in tegra_smmu_debugfs_exit()  argument
1074 	debugfs_remove_recursive(smmu->debugfs);  in tegra_smmu_debugfs_exit()
1081 	struct tegra_smmu *smmu;  in tegra_smmu_probe()  local
1086 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);  in tegra_smmu_probe()
1087 	if (!smmu)  in tegra_smmu_probe()
1098 	mc->smmu = smmu;  in tegra_smmu_probe()
1102 	smmu->asids = devm_kzalloc(dev, size, GFP_KERNEL);  in tegra_smmu_probe()
1103 	if (!smmu->asids)  in tegra_smmu_probe()
1106 	INIT_LIST_HEAD(&smmu->groups);  in tegra_smmu_probe()
1107 	mutex_init(&smmu->lock);  in tegra_smmu_probe()
1109 	smmu->regs = mc->regs;  in tegra_smmu_probe()
1110 	smmu->soc = soc;  in tegra_smmu_probe()
1111 	smmu->dev = dev;  in tegra_smmu_probe()
1112 	smmu->mc = mc;  in tegra_smmu_probe()
1114 	smmu->pfn_mask =  in tegra_smmu_probe()
1117 		mc->soc->num_address_bits, smmu->pfn_mask);  in tegra_smmu_probe()
1118 	smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;  in tegra_smmu_probe()
1119 	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,  in tegra_smmu_probe()
1120 		smmu->tlb_mask);  in tegra_smmu_probe()
1127 	smmu_writel(smmu, value, SMMU_PTC_CONFIG);  in tegra_smmu_probe()
1130 		SMMU_TLB_CONFIG_ACTIVE_LINES(smmu);  in tegra_smmu_probe()
1135 	smmu_writel(smmu, value, SMMU_TLB_CONFIG);  in tegra_smmu_probe()
1137 	smmu_flush_ptc_all(smmu);  in tegra_smmu_probe()
1138 	smmu_flush_tlb(smmu);  in tegra_smmu_probe()
1139 	smmu_writel(smmu, SMMU_CONFIG_ENABLE, SMMU_CONFIG);  in tegra_smmu_probe()
1140 	smmu_flush(smmu);  in tegra_smmu_probe()
1144 	err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));  in tegra_smmu_probe()
1148 	err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev);  in tegra_smmu_probe()
1163 		tegra_smmu_debugfs_init(smmu);  in tegra_smmu_probe()
1165 	return smmu;  in tegra_smmu_probe()
1170 	iommu_device_unregister(&smmu->iommu);  in tegra_smmu_probe()
1172 	iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_probe()
1177 void tegra_smmu_remove(struct tegra_smmu *smmu)  in tegra_smmu_remove()  argument
1179 	iommu_device_unregister(&smmu->iommu);  in tegra_smmu_remove()
1180 	iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_remove()
1183 		tegra_smmu_debugfs_exit(smmu);  in tegra_smmu_remove()