1 // SPDX-License-Identifier: GPL-2.0-only
3  * Copyright (C) 2011-2014 NVIDIA CORPORATION.  All rights reserved.
17 #include <linux/dma-mapping.h>
19 #include <soc/tegra/ahb.h>
20 #include <soc/tegra/mc.h>
25 	const struct tegra_smmu_group_soc *soc;  member
35 	const struct tegra_smmu_soc *soc;  member
73 	writel(value, smmu->regs + offset);  in smmu_writel()
78 	return readl(smmu->regs + offset);  in smmu_readl()
88 	((smmu)->soc->num_tlb_lines & (smmu)->tlb_mask)
120 /* per-SWGROUP SMMU_*_ASID register */
135 #define SMMU_PAGE_MASK		(~(SMMU_SIZE_PT-1))
158 	return (iova >> SMMU_PDE_SHIFT) & (SMMU_NUM_PDE - 1);  in iova_pd_index()
163 	return (iova >> SMMU_PTE_SHIFT) & (SMMU_NUM_PTE - 1);  in iova_pt_index()
169 	return (addr & smmu->pfn_mask) == addr;  in smmu_dma_addr_valid()
174 	return (dma_addr_t)(pde & smmu->pfn_mask) << 12;  in smmu_pde_to_dma()
187 	offset &= ~(smmu->mc->soc->atom_size - 1);  in smmu_flush_ptc()
189 	if (smmu->mc->soc->num_address_bits > 32) {  in smmu_flush_ptc()
212 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_asid()
227 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_section()
242 	if (smmu->soc->num_asids == 4)  in smmu_flush_tlb_group()
260 	id = find_first_zero_bit(smmu->asids, smmu->soc->num_asids);  in tegra_smmu_alloc_asid()
261 	if (id >= smmu->soc->num_asids)  in tegra_smmu_alloc_asid()
262 		return -ENOSPC;  in tegra_smmu_alloc_asid()
264 	set_bit(id, smmu->asids);  in tegra_smmu_alloc_asid()
272 	clear_bit(id, smmu->asids);  in tegra_smmu_free_asid()
286 	as->attr = SMMU_PD_READABLE | SMMU_PD_WRITABLE | SMMU_PD_NONSECURE;  in tegra_smmu_domain_alloc()
288 	as->pd = alloc_page(GFP_KERNEL | __GFP_DMA | __GFP_ZERO);  in tegra_smmu_domain_alloc()
289 	if (!as->pd) {  in tegra_smmu_domain_alloc()
294 	as->count = kcalloc(SMMU_NUM_PDE, sizeof(u32), GFP_KERNEL);  in tegra_smmu_domain_alloc()
295 	if (!as->count) {  in tegra_smmu_domain_alloc()
296 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
301 	as->pts = kcalloc(SMMU_NUM_PDE, sizeof(*as->pts), GFP_KERNEL);  in tegra_smmu_domain_alloc()
302 	if (!as->pts) {  in tegra_smmu_domain_alloc()
303 		kfree(as->count);  in tegra_smmu_domain_alloc()
304 		__free_page(as->pd);  in tegra_smmu_domain_alloc()
309 	spin_lock_init(&as->lock);  in tegra_smmu_domain_alloc()
312 	as->domain.geometry.aperture_start = 0;  in tegra_smmu_domain_alloc()
313 	as->domain.geometry.aperture_end = 0xffffffff;  in tegra_smmu_domain_alloc()
314 	as->domain.geometry.force_aperture = true;  in tegra_smmu_domain_alloc()
316 	return &as->domain;  in tegra_smmu_domain_alloc()
325 	WARN_ON_ONCE(as->use_count);  in tegra_smmu_domain_free()
326 	kfree(as->count);  in tegra_smmu_domain_free()
327 	kfree(as->pts);  in tegra_smmu_domain_free()
337 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_find_swgroup()
338 		if (smmu->soc->swgroups[i].swgroup == swgroup) {  in tegra_smmu_find_swgroup()
339 			group = &smmu->soc->swgroups[i];  in tegra_smmu_find_swgroup()
356 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_enable()
360 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_enable()
362 		pr_warn("%s group from swgroup %u not found\n", __func__,  in tegra_smmu_enable()
368 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_enable()
369 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_enable()
371 		if (client->swgroup != swgroup)  in tegra_smmu_enable()
374 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_enable()
375 		value |= BIT(client->regs.smmu.bit);  in tegra_smmu_enable()
376 		smmu_writel(smmu, value, client->regs.smmu.reg);  in tegra_smmu_enable()
389 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_disable()
393 		smmu_writel(smmu, value, group->reg);  in tegra_smmu_disable()
396 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_disable()
397 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_disable()
399 		if (client->swgroup != swgroup)  in tegra_smmu_disable()
402 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_disable()
403 		value &= ~BIT(client->regs.smmu.bit);  in tegra_smmu_disable()
404 		smmu_writel(smmu, value, client->regs.smmu.reg);  in tegra_smmu_disable()
414 	mutex_lock(&smmu->lock);  in tegra_smmu_as_prepare()
416 	if (as->use_count > 0) {  in tegra_smmu_as_prepare()
417 		as->use_count++;  in tegra_smmu_as_prepare()
421 	as->pd_dma = dma_map_page(smmu->dev, as->pd, 0, SMMU_SIZE_PD,  in tegra_smmu_as_prepare()
423 	if (dma_mapping_error(smmu->dev, as->pd_dma)) {  in tegra_smmu_as_prepare()
424 		err = -ENOMEM;  in tegra_smmu_as_prepare()
428 	/* We can't handle 64-bit DMA addresses */  in tegra_smmu_as_prepare()
429 	if (!smmu_dma_addr_valid(smmu, as->pd_dma)) {  in tegra_smmu_as_prepare()
430 		err = -ENOMEM;  in tegra_smmu_as_prepare()
434 	err = tegra_smmu_alloc_asid(smmu, &as->id);  in tegra_smmu_as_prepare()
438 	smmu_flush_ptc(smmu, as->pd_dma, 0);  in tegra_smmu_as_prepare()
439 	smmu_flush_tlb_asid(smmu, as->id);  in tegra_smmu_as_prepare()
441 	smmu_writel(smmu, as->id & 0x7f, SMMU_PTB_ASID);  in tegra_smmu_as_prepare()
442 	value = SMMU_PTB_DATA_VALUE(as->pd_dma, as->attr);  in tegra_smmu_as_prepare()
446 	as->smmu = smmu;  in tegra_smmu_as_prepare()
447 	as->use_count++;  in tegra_smmu_as_prepare()
449 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_prepare()
454 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_prepare()
456 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_prepare()
464 	mutex_lock(&smmu->lock);  in tegra_smmu_as_unprepare()
466 	if (--as->use_count > 0) {  in tegra_smmu_as_unprepare()
467 		mutex_unlock(&smmu->lock);  in tegra_smmu_as_unprepare()
471 	tegra_smmu_free_asid(smmu, as->id);  in tegra_smmu_as_unprepare()
473 	dma_unmap_page(smmu->dev, as->pd_dma, SMMU_SIZE_PD, DMA_TO_DEVICE);  in tegra_smmu_as_unprepare()
475 	as->smmu = NULL;  in tegra_smmu_as_unprepare()
477 	mutex_unlock(&smmu->lock);  in tegra_smmu_as_unprepare()
490 		return -ENOENT;  in tegra_smmu_attach_dev()
492 	for (index = 0; index < fwspec->num_ids; index++) {  in tegra_smmu_attach_dev()
497 		tegra_smmu_enable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
501 		return -ENODEV;  in tegra_smmu_attach_dev()
506 	while (index--) {  in tegra_smmu_attach_dev()
507 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_attach_dev()
518 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_detach_dev()
524 	for (index = 0; index < fwspec->num_ids; index++) {  in tegra_smmu_detach_dev()
525 		tegra_smmu_disable(smmu, fwspec->ids[index], as->id);  in tegra_smmu_detach_dev()
534 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pde()
535 	u32 *pd = page_address(as->pd);  in tegra_smmu_set_pde()
542 	dma_sync_single_range_for_device(smmu->dev, as->pd_dma, offset,  in tegra_smmu_set_pde()
546 	smmu_flush_ptc(smmu, as->pd_dma, offset);  in tegra_smmu_set_pde()
547 	smmu_flush_tlb_section(smmu, as->id, iova);  in tegra_smmu_set_pde()
562 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_lookup()
566 	pt_page = as->pts[pd_index];  in tegra_smmu_pte_lookup()
570 	pd = page_address(as->pd);  in tegra_smmu_pte_lookup()
580 	struct tegra_smmu *smmu = as->smmu;  in as_get_pte()
582 	if (!as->pts[pde]) {  in as_get_pte()
585 		dma = dma_map_page(smmu->dev, page, 0, SMMU_SIZE_PT,  in as_get_pte()
587 		if (dma_mapping_error(smmu->dev, dma)) {  in as_get_pte()
593 			dma_unmap_page(smmu->dev, dma, SMMU_SIZE_PT,  in as_get_pte()
599 		as->pts[pde] = page;  in as_get_pte()
606 		u32 *pd = page_address(as->pd);  in as_get_pte()
611 	return tegra_smmu_pte_offset(as->pts[pde], iova);  in as_get_pte()
618 	as->count[pd_index]++;  in tegra_smmu_pte_get_use()
624 	struct page *page = as->pts[pde];  in tegra_smmu_pte_put_use()
630 	if (--as->count[pde] == 0) {  in tegra_smmu_pte_put_use()
631 		struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_pte_put_use()
632 		u32 *pd = page_address(as->pd);  in tegra_smmu_pte_put_use()
637 		dma_unmap_page(smmu->dev, pte_dma, SMMU_SIZE_PT, DMA_TO_DEVICE);  in tegra_smmu_pte_put_use()
639 		as->pts[pde] = NULL;  in tegra_smmu_pte_put_use()
646 	struct tegra_smmu *smmu = as->smmu;  in tegra_smmu_set_pte()
651 	dma_sync_single_range_for_device(smmu->dev, pte_dma, offset,  in tegra_smmu_set_pte()
654 	smmu_flush_tlb_group(smmu, as->id, iova);  in tegra_smmu_set_pte()
663 	struct page *page = as->pts[pde];  in as_get_pde_page()
672 	 * spinlock needs to be unlocked and re-locked after allocation.  in as_get_pde_page()
675 		spin_unlock_irqrestore(&as->lock, *flags);  in as_get_pde_page()
680 		spin_lock_irqsave(&as->lock, *flags);  in as_get_pde_page()
687 	if (as->pts[pde]) {  in as_get_pde_page()
691 		page = as->pts[pde];  in as_get_pde_page()
710 		return -ENOMEM;  in __tegra_smmu_map()
714 		return -ENOMEM;  in __tegra_smmu_map()
716 	/* If we aren't overwriting a pre-existing entry, increment use */  in __tegra_smmu_map()
759 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_map()
761 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_map()
772 	spin_lock_irqsave(&as->lock, flags);  in tegra_smmu_unmap()
774 	spin_unlock_irqrestore(&as->lock, flags);  in tegra_smmu_unmap()
791 	pfn = *pte & as->smmu->pfn_mask;  in tegra_smmu_iova_to_phys()
807 		put_device(&pdev->dev);  in tegra_smmu_find()
811 	return mc->smmu;  in tegra_smmu_find()
817 	const struct iommu_ops *ops = smmu->iommu.ops;  in tegra_smmu_configure()
820 	err = iommu_fwspec_init(dev, &dev->of_node->fwnode, ops);  in tegra_smmu_configure()
826 	err = ops->of_xlate(dev, args);  in tegra_smmu_configure()
838 	struct device_node *np = dev->of_node;  in tegra_smmu_probe_device()
844 	while (of_parse_phandle_with_args(np, "iommus", "#iommu-cells", index,  in tegra_smmu_probe_device()
862 		return ERR_PTR(-ENODEV);  in tegra_smmu_probe_device()
864 	return &smmu->iommu;  in tegra_smmu_probe_device()
872 	for (i = 0; i < smmu->soc->num_groups; i++)  in tegra_smmu_find_group()
873 		for (j = 0; j < smmu->soc->groups[i].num_swgroups; j++)  in tegra_smmu_find_group()
874 			if (smmu->soc->groups[i].swgroups[j] == swgroup)  in tegra_smmu_find_group()
875 				return &smmu->soc->groups[i];  in tegra_smmu_find_group()
883 	struct tegra_smmu *smmu = group->smmu;  in tegra_smmu_group_release()
885 	mutex_lock(&smmu->lock);  in tegra_smmu_group_release()
886 	list_del(&group->list);  in tegra_smmu_group_release()
887 	mutex_unlock(&smmu->lock);  in tegra_smmu_group_release()
894 	const struct tegra_smmu_group_soc *soc;  in tegra_smmu_device_group()  local
895 	unsigned int swgroup = fwspec->ids[0];  in tegra_smmu_device_group()
900 	soc = tegra_smmu_find_group(smmu, swgroup);  in tegra_smmu_device_group()
902 	mutex_lock(&smmu->lock);  in tegra_smmu_device_group()
905 	list_for_each_entry(group, &smmu->groups, list)  in tegra_smmu_device_group()
906 		if ((group->swgroup == swgroup) || (soc && group->soc == soc)) {  in tegra_smmu_device_group()
907 			grp = iommu_group_ref_get(group->group);  in tegra_smmu_device_group()
908 			mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
912 	group = devm_kzalloc(smmu->dev, sizeof(*group), GFP_KERNEL);  in tegra_smmu_device_group()
914 		mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
918 	INIT_LIST_HEAD(&group->list);  in tegra_smmu_device_group()
919 	group->swgroup = swgroup;  in tegra_smmu_device_group()
920 	group->smmu = smmu;  in tegra_smmu_device_group()
921 	group->soc = soc;  in tegra_smmu_device_group()
924 		group->group = pci_device_group(dev);  in tegra_smmu_device_group()
926 		group->group = generic_device_group(dev);  in tegra_smmu_device_group()
928 	if (IS_ERR(group->group)) {  in tegra_smmu_device_group()
929 		devm_kfree(smmu->dev, group);  in tegra_smmu_device_group()
930 		mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
934 	iommu_group_set_iommudata(group->group, group, tegra_smmu_group_release);  in tegra_smmu_device_group()
935 	if (soc)  in tegra_smmu_device_group()
936 		iommu_group_set_name(group->group, soc->name);  in tegra_smmu_device_group()
937 	list_add_tail(&group->list, &smmu->groups);  in tegra_smmu_device_group()
938 	mutex_unlock(&smmu->lock);  in tegra_smmu_device_group()
940 	return group->group;  in tegra_smmu_device_group()
946 	struct platform_device *iommu_pdev = of_find_device_by_node(args->np);  in tegra_smmu_of_xlate()
948 	u32 id = args->args[0];  in tegra_smmu_of_xlate()
951 	 * Note: we are here releasing the reference of &iommu_pdev->dev, which  in tegra_smmu_of_xlate()
952 	 * is mc->dev. Although some functions in tegra_smmu_ops may keep using  in tegra_smmu_of_xlate()
953 	 * its private data beyond this point, it's still safe to do so because  in tegra_smmu_of_xlate()
957 	put_device(&iommu_pdev->dev);  in tegra_smmu_of_xlate()
959 	dev_iommu_priv_set(dev, mc->smmu);  in tegra_smmu_of_xlate()
983 		{ .compatible = "nvidia,tegra30-ahb", },  in tegra_smmu_ahb_enable()
995 static int tegra_smmu_swgroups_show(struct seq_file *s, void *data)  in tegra_smmu_swgroups_show()  argument
997 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_swgroups_show()
1001 	seq_printf(s, "swgroup    enabled  ASID\n");  in tegra_smmu_swgroups_show()
1002 	seq_printf(s, "------------------------\n");  in tegra_smmu_swgroups_show()
1004 	for (i = 0; i < smmu->soc->num_swgroups; i++) {  in tegra_smmu_swgroups_show()
1005 		const struct tegra_smmu_swgroup *group = &smmu->soc->swgroups[i];  in tegra_smmu_swgroups_show()
1009 		value = smmu_readl(smmu, group->reg);  in tegra_smmu_swgroups_show()
1018 		seq_printf(s, "%-9s  %-7s  %#04x\n", group->name, status,  in tegra_smmu_swgroups_show()
1027 static int tegra_smmu_clients_show(struct seq_file *s, void *data)  in tegra_smmu_clients_show()  argument
1029 	struct tegra_smmu *smmu = s->private;  in tegra_smmu_clients_show()
1033 	seq_printf(s, "client       enabled\n");  in tegra_smmu_clients_show()
1034 	seq_printf(s, "--------------------\n");  in tegra_smmu_clients_show()
1036 	for (i = 0; i < smmu->soc->num_clients; i++) {  in tegra_smmu_clients_show()
1037 		const struct tegra_mc_client *client = &smmu->soc->clients[i];  in tegra_smmu_clients_show()
1040 		value = smmu_readl(smmu, client->regs.smmu.reg);  in tegra_smmu_clients_show()
1042 		if (value & BIT(client->regs.smmu.bit))  in tegra_smmu_clients_show()
1047 		seq_printf(s, "%-12s %s\n", client->name, status);  in tegra_smmu_clients_show()
1057 	smmu->debugfs = debugfs_create_dir("smmu", NULL);  in tegra_smmu_debugfs_init()
1058 	if (!smmu->debugfs)  in tegra_smmu_debugfs_init()
1061 	debugfs_create_file("swgroups", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1063 	debugfs_create_file("clients", S_IRUGO, smmu->debugfs, smmu,  in tegra_smmu_debugfs_init()
1069 	debugfs_remove_recursive(smmu->debugfs);  in tegra_smmu_debugfs_exit()
1073 				    const struct tegra_smmu_soc *soc,  in tegra_smmu_probe()  argument
1082 		return ERR_PTR(-ENOMEM);  in tegra_smmu_probe()
1090 	 * callback via the IOMMU device's .drvdata field.  in tegra_smmu_probe()
1092 	mc->smmu = smmu;  in tegra_smmu_probe()
1094 	smmu->asids = devm_bitmap_zalloc(dev, soc->num_asids, GFP_KERNEL);  in tegra_smmu_probe()
1095 	if (!smmu->asids)  in tegra_smmu_probe()
1096 		return ERR_PTR(-ENOMEM);  in tegra_smmu_probe()
1098 	INIT_LIST_HEAD(&smmu->groups);  in tegra_smmu_probe()
1099 	mutex_init(&smmu->lock);  in tegra_smmu_probe()
1101 	smmu->regs = mc->regs;  in tegra_smmu_probe()
1102 	smmu->soc = soc;  in tegra_smmu_probe()
1103 	smmu->dev = dev;  in tegra_smmu_probe()
1104 	smmu->mc = mc;  in tegra_smmu_probe()
1106 	smmu->pfn_mask =  in tegra_smmu_probe()
1107 		BIT_MASK(mc->soc->num_address_bits - SMMU_PTE_SHIFT) - 1;  in tegra_smmu_probe()
1109 		mc->soc->num_address_bits, smmu->pfn_mask);  in tegra_smmu_probe()
1110 	smmu->tlb_mask = (1 << fls(smmu->soc->num_tlb_lines)) - 1;  in tegra_smmu_probe()
1111 	dev_dbg(dev, "TLB lines: %u, mask: %#lx\n", smmu->soc->num_tlb_lines,  in tegra_smmu_probe()
1112 		smmu->tlb_mask);  in tegra_smmu_probe()
1116 	if (soc->supports_request_limit)  in tegra_smmu_probe()
1124 	if (soc->supports_round_robin_arbitration)  in tegra_smmu_probe()
1136 	err = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, dev_name(dev));  in tegra_smmu_probe()
1140 	err = iommu_device_register(&smmu->iommu, &tegra_smmu_ops, dev);  in tegra_smmu_probe()
1142 		iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_probe()
1154 	iommu_device_unregister(&smmu->iommu);  in tegra_smmu_remove()
1155 	iommu_device_sysfs_remove(&smmu->iommu);  in tegra_smmu_remove()