1 // SPDX-License-Identifier: GPL-2.0-only
8 #include <linux/dma-mapping.h>
25 	{ .compatible = "nvidia,tegra20-mc-gart", .data = &tegra20_mc_soc },
28 	{ .compatible = "nvidia,tegra30-mc", .data = &tegra30_mc_soc },
31 	{ .compatible = "nvidia,tegra114-mc", .data = &tegra114_mc_soc },
34 	{ .compatible = "nvidia,tegra124-mc", .data = &tegra124_mc_soc },
37 	{ .compatible = "nvidia,tegra132-mc", .data = &tegra132_mc_soc },
40 	{ .compatible = "nvidia,tegra210-mc", .data = &tegra210_mc_soc },
43 	{ .compatible = "nvidia,tegra186-mc", .data = &tegra186_mc_soc },
46 	{ .compatible = "nvidia,tegra194-mc", .data = &tegra194_mc_soc },
56 	put_device(mc->dev);  in tegra_mc_devm_action_put_device()
60  * devm_tegra_memory_controller_get() - get Tegra Memory Controller handle
63  * This function will search for the Memory Controller node in a device-tree
75 	np = of_parse_phandle(dev->of_node, "nvidia,memory-controller", 0);  in devm_tegra_memory_controller_get()
77 		return ERR_PTR(-ENOENT);  in devm_tegra_memory_controller_get()
82 		return ERR_PTR(-ENODEV);  in devm_tegra_memory_controller_get()
86 		put_device(&pdev->dev);  in devm_tegra_memory_controller_get()
87 		return ERR_PTR(-EPROBE_DEFER);  in devm_tegra_memory_controller_get()
92 		put_device(mc->dev);  in devm_tegra_memory_controller_get()
102 	if (mc->soc->ops && mc->soc->ops->probe_device)  in tegra_mc_probe_device()
103 		return mc->soc->ops->probe_device(mc, dev);  in tegra_mc_probe_device()
115 	spin_lock_irqsave(&mc->lock, flags);  in tegra_mc_block_dma_common()
117 	value = mc_readl(mc, rst->control) | BIT(rst->bit);  in tegra_mc_block_dma_common()
118 	mc_writel(mc, value, rst->control);  in tegra_mc_block_dma_common()
120 	spin_unlock_irqrestore(&mc->lock, flags);  in tegra_mc_block_dma_common()
128 	return (mc_readl(mc, rst->status) & BIT(rst->bit)) != 0;  in tegra_mc_dma_idling_common()
137 	spin_lock_irqsave(&mc->lock, flags);  in tegra_mc_unblock_dma_common()
139 	value = mc_readl(mc, rst->control) & ~BIT(rst->bit);  in tegra_mc_unblock_dma_common()
140 	mc_writel(mc, value, rst->control);  in tegra_mc_unblock_dma_common()
142 	spin_unlock_irqrestore(&mc->lock, flags);  in tegra_mc_unblock_dma_common()
150 	return (mc_readl(mc, rst->control) & BIT(rst->bit)) != 0;  in tegra_mc_reset_status_common()
170 	for (i = 0; i < mc->soc->num_resets; i++)  in tegra_mc_reset_find()
171 		if (mc->soc->resets[i].id == id)  in tegra_mc_reset_find()
172 			return &mc->soc->resets[i];  in tegra_mc_reset_find()
188 		return -ENODEV;  in tegra_mc_hotreset_assert()
190 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_assert()
192 		return -ENODEV;  in tegra_mc_hotreset_assert()
195 	if (rst_ops->reset_status) {  in tegra_mc_hotreset_assert()
197 		if (rst_ops->reset_status(mc, rst))  in tegra_mc_hotreset_assert()
201 	if (rst_ops->block_dma) {  in tegra_mc_hotreset_assert()
203 		err = rst_ops->block_dma(mc, rst);  in tegra_mc_hotreset_assert()
205 			dev_err(mc->dev, "failed to block %s DMA: %d\n",  in tegra_mc_hotreset_assert()
206 				rst->name, err);  in tegra_mc_hotreset_assert()
211 	if (rst_ops->dma_idling) {  in tegra_mc_hotreset_assert()
213 		while (!rst_ops->dma_idling(mc, rst)) {  in tegra_mc_hotreset_assert()
214 			if (!retries--) {  in tegra_mc_hotreset_assert()
215 				dev_err(mc->dev, "failed to flush %s DMA\n",  in tegra_mc_hotreset_assert()
216 					rst->name);  in tegra_mc_hotreset_assert()
217 				return -EBUSY;  in tegra_mc_hotreset_assert()
224 	if (rst_ops->hotreset_assert) {  in tegra_mc_hotreset_assert()
226 		err = rst_ops->hotreset_assert(mc, rst);  in tegra_mc_hotreset_assert()
228 			dev_err(mc->dev, "failed to hot reset %s: %d\n",  in tegra_mc_hotreset_assert()
229 				rst->name, err);  in tegra_mc_hotreset_assert()
247 		return -ENODEV;  in tegra_mc_hotreset_deassert()
249 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_deassert()
251 		return -ENODEV;  in tegra_mc_hotreset_deassert()
253 	if (rst_ops->hotreset_deassert) {  in tegra_mc_hotreset_deassert()
255 		err = rst_ops->hotreset_deassert(mc, rst);  in tegra_mc_hotreset_deassert()
257 			dev_err(mc->dev, "failed to deassert hot reset %s: %d\n",  in tegra_mc_hotreset_deassert()
258 				rst->name, err);  in tegra_mc_hotreset_deassert()
263 	if (rst_ops->unblock_dma) {  in tegra_mc_hotreset_deassert()
265 		err = rst_ops->unblock_dma(mc, rst);  in tegra_mc_hotreset_deassert()
267 			dev_err(mc->dev, "failed to unblock %s DMA : %d\n",  in tegra_mc_hotreset_deassert()
268 				rst->name, err);  in tegra_mc_hotreset_deassert()
285 		return -ENODEV;  in tegra_mc_hotreset_status()
287 	rst_ops = mc->soc->reset_ops;  in tegra_mc_hotreset_status()
289 		return -ENODEV;  in tegra_mc_hotreset_status()
291 	return rst_ops->reset_status(mc, rst);  in tegra_mc_hotreset_status()
304 	mc->reset.ops = &tegra_mc_reset_ops;  in tegra_mc_reset_setup()
305 	mc->reset.owner = THIS_MODULE;  in tegra_mc_reset_setup()
306 	mc->reset.of_node = mc->dev->of_node;  in tegra_mc_reset_setup()
307 	mc->reset.of_reset_n_cells = 1;  in tegra_mc_reset_setup()
308 	mc->reset.nr_resets = mc->soc->num_resets;  in tegra_mc_reset_setup()
310 	err = reset_controller_register(&mc->reset);  in tegra_mc_reset_setup()
322 	for (i = 0; i < mc->num_timings; i++) {  in tegra_mc_write_emem_configuration()
323 		if (mc->timings[i].rate == rate) {  in tegra_mc_write_emem_configuration()
324 			timing = &mc->timings[i];  in tegra_mc_write_emem_configuration()
330 		dev_err(mc->dev, "no memory timing registered for rate %lu\n",  in tegra_mc_write_emem_configuration()
332 		return -EINVAL;  in tegra_mc_write_emem_configuration()
335 	for (i = 0; i < mc->soc->num_emem_regs; ++i)  in tegra_mc_write_emem_configuration()
336 		mc_writel(mc, timing->emem_data[i], mc->soc->emem_regs[i]);  in tegra_mc_write_emem_configuration()
366 	tick = (unsigned long long)mc->tick * clk_get_rate(mc->clk);  in tegra_mc_setup_latency_allowance()
375 	for (i = 0; i < mc->soc->num_clients; i++) {  in tegra_mc_setup_latency_allowance()
376 		const struct tegra_mc_client *client = &mc->soc->clients[i];  in tegra_mc_setup_latency_allowance()
379 		value = mc_readl(mc, client->regs.la.reg);  in tegra_mc_setup_latency_allowance()
380 		value &= ~(client->regs.la.mask << client->regs.la.shift);  in tegra_mc_setup_latency_allowance()
381 		value |= (client->regs.la.def & client->regs.la.mask) << client->regs.la.shift;  in tegra_mc_setup_latency_allowance()
382 		mc_writel(mc, value, client->regs.la.reg);  in tegra_mc_setup_latency_allowance()
398 	err = of_property_read_u32(node, "clock-frequency", &tmp);  in load_one_timing()
400 		dev_err(mc->dev,  in load_one_timing()
405 	timing->rate = tmp;  in load_one_timing()
406 	timing->emem_data = devm_kcalloc(mc->dev, mc->soc->num_emem_regs,  in load_one_timing()
408 	if (!timing->emem_data)  in load_one_timing()
409 		return -ENOMEM;  in load_one_timing()
411 	err = of_property_read_u32_array(node, "nvidia,emem-configuration",  in load_one_timing()
412 					 timing->emem_data,  in load_one_timing()
413 					 mc->soc->num_emem_regs);  in load_one_timing()
415 		dev_err(mc->dev,  in load_one_timing()
431 	mc->timings = devm_kcalloc(mc->dev, child_count, sizeof(*timing),  in load_timings()
433 	if (!mc->timings)  in load_timings()
434 		return -ENOMEM;  in load_timings()
436 	mc->num_timings = child_count;  in load_timings()
439 		timing = &mc->timings[i++];  in load_timings()
459 	mc->num_timings = 0;  in tegra_mc_setup_timings()
461 	for_each_child_of_node(mc->dev->of_node, node) {  in tegra_mc_setup_timings()
462 		err = of_property_read_u32(node, "nvidia,ram-code",  in tegra_mc_setup_timings()
474 	if (mc->num_timings == 0)  in tegra_mc_setup_timings()
475 		dev_warn(mc->dev,  in tegra_mc_setup_timings()
486 	mc->clk = devm_clk_get_optional(mc->dev, "mc");  in tegra30_mc_probe()
487 	if (IS_ERR(mc->clk)) {  in tegra30_mc_probe()
488 		dev_err(mc->dev, "failed to get MC clock: %ld\n", PTR_ERR(mc->clk));  in tegra30_mc_probe()
489 		return PTR_ERR(mc->clk);  in tegra30_mc_probe()
497 		dev_err(mc->dev, "failed to setup latency allowance: %d\n", err);  in tegra30_mc_probe()
503 		dev_err(mc->dev, "failed to setup timings: %d\n", err);  in tegra30_mc_probe()
517 	status = mc_readl(mc, MC_INTSTATUS) & mc->soc->intmask;  in tegra30_mc_handle_irq()
534 		if (mc->soc->num_address_bits > 32) {  in tegra30_mc_handle_irq()
551 		id = value & mc->soc->client_id_mask;  in tegra30_mc_handle_irq()
553 		for (i = 0; i < mc->soc->num_clients; i++) {  in tegra30_mc_handle_irq()
554 			if (mc->soc->clients[i].id == id) {  in tegra30_mc_handle_irq()
555 				client = mc->soc->clients[i].name;  in tegra30_mc_handle_irq()
572 				perm[2] = '-';  in tegra30_mc_handle_irq()
577 				perm[3] = '-';  in tegra30_mc_handle_irq()
580 				perm[4] = '-';  in tegra30_mc_handle_irq()
596 		dev_err_ratelimited(mc->dev, "%s: %s%s @%pa: %s (%s%s)\n",  in tegra30_mc_handle_irq()
637  * up to the External Memory Controller (EMC) interconnect provider which
638  * re-configures hardware interface to External Memory (EMEM) in accordance
644  *               +----+
645  * +--------+    |    |
646  * | TEXSRD +--->+    |
647  * +--------+    |    |
648  *               |    |    +-----+    +------+
649  *    ...        | MC +--->+ EMC +--->+ EMEM |
650  *               |    |    +-----+    +------+
651  * +--------+    |    |
652  * | DISP.. +--->+    |
653  * +--------+    |    |
654  *               +----+
662 	/* older device-trees don't have interconnect properties */  in tegra_mc_interconnect_setup()
663 	if (!device_property_present(mc->dev, "#interconnect-cells") ||  in tegra_mc_interconnect_setup()
664 	    !mc->soc->icc_ops)  in tegra_mc_interconnect_setup()
667 	mc->provider.dev = mc->dev;  in tegra_mc_interconnect_setup()
668 	mc->provider.data = &mc->provider;  in tegra_mc_interconnect_setup()
669 	mc->provider.set = mc->soc->icc_ops->set;  in tegra_mc_interconnect_setup()
670 	mc->provider.aggregate = mc->soc->icc_ops->aggregate;  in tegra_mc_interconnect_setup()
671 	mc->provider.xlate_extended = mc->soc->icc_ops->xlate_extended;  in tegra_mc_interconnect_setup()
673 	err = icc_provider_add(&mc->provider);  in tegra_mc_interconnect_setup()
684 	node->name = "Memory Controller";  in tegra_mc_interconnect_setup()
685 	icc_node_add(node, &mc->provider);  in tegra_mc_interconnect_setup()
692 	for (i = 0; i < mc->soc->num_clients; i++) {  in tegra_mc_interconnect_setup()
694 		node = icc_node_create(mc->soc->clients[i].id);  in tegra_mc_interconnect_setup()
700 		node->name = mc->soc->clients[i].name;  in tegra_mc_interconnect_setup()
701 		icc_node_add(node, &mc->provider);  in tegra_mc_interconnect_setup()
712 	 * since syncing works for the EMC drivers, hence we can sync the  in tegra_mc_interconnect_setup()
713 	 * MC driver by ourselves and then EMC will complete syncing of  in tegra_mc_interconnect_setup()
716 	icc_sync_state(mc->dev);  in tegra_mc_interconnect_setup()
721 	icc_nodes_remove(&mc->provider);  in tegra_mc_interconnect_setup()
723 	icc_provider_del(&mc->provider);  in tegra_mc_interconnect_setup()
735 	mc = devm_kzalloc(&pdev->dev, sizeof(*mc), GFP_KERNEL);  in tegra_mc_probe()
737 		return -ENOMEM;  in tegra_mc_probe()
740 	spin_lock_init(&mc->lock);  in tegra_mc_probe()
741 	mc->soc = of_device_get_match_data(&pdev->dev);  in tegra_mc_probe()
742 	mc->dev = &pdev->dev;  in tegra_mc_probe()
744 	mask = DMA_BIT_MASK(mc->soc->num_address_bits);  in tegra_mc_probe()
746 	err = dma_coerce_mask_and_coherent(&pdev->dev, mask);  in tegra_mc_probe()
748 		dev_err(&pdev->dev, "failed to set DMA mask: %d\n", err);  in tegra_mc_probe()
753 	mc->tick = 30;  in tegra_mc_probe()
756 	mc->regs = devm_ioremap_resource(&pdev->dev, res);  in tegra_mc_probe()
757 	if (IS_ERR(mc->regs))  in tegra_mc_probe()
758 		return PTR_ERR(mc->regs);  in tegra_mc_probe()
760 	mc->debugfs.root = debugfs_create_dir("mc", NULL);  in tegra_mc_probe()
762 	if (mc->soc->ops && mc->soc->ops->probe) {  in tegra_mc_probe()
763 		err = mc->soc->ops->probe(mc);  in tegra_mc_probe()
768 	if (mc->soc->ops && mc->soc->ops->handle_irq) {  in tegra_mc_probe()
769 		mc->irq = platform_get_irq(pdev, 0);  in tegra_mc_probe()
770 		if (mc->irq < 0)  in tegra_mc_probe()
771 			return mc->irq;  in tegra_mc_probe()
773 		WARN(!mc->soc->client_id_mask, "missing client ID mask for this SoC\n");  in tegra_mc_probe()
775 		mc_writel(mc, mc->soc->intmask, MC_INTMASK);  in tegra_mc_probe()
777 		err = devm_request_irq(&pdev->dev, mc->irq, mc->soc->ops->handle_irq, 0,  in tegra_mc_probe()
778 				       dev_name(&pdev->dev), mc);  in tegra_mc_probe()
780 			dev_err(&pdev->dev, "failed to request IRQ#%u: %d\n", mc->irq,  in tegra_mc_probe()
786 	if (mc->soc->reset_ops) {  in tegra_mc_probe()
789 			dev_err(&pdev->dev, "failed to register reset controller: %d\n", err);  in tegra_mc_probe()
794 		dev_err(&pdev->dev, "failed to initialize interconnect: %d\n",  in tegra_mc_probe()
797 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_SMMU) && mc->soc->smmu) {  in tegra_mc_probe()
798 		mc->smmu = tegra_smmu_probe(&pdev->dev, mc->soc->smmu, mc);  in tegra_mc_probe()
799 		if (IS_ERR(mc->smmu)) {  in tegra_mc_probe()
800 			dev_err(&pdev->dev, "failed to probe SMMU: %ld\n",  in tegra_mc_probe()
801 				PTR_ERR(mc->smmu));  in tegra_mc_probe()
802 			mc->smmu = NULL;  in tegra_mc_probe()
806 	if (IS_ENABLED(CONFIG_TEGRA_IOMMU_GART) && !mc->soc->smmu) {  in tegra_mc_probe()
807 		mc->gart = tegra_gart_probe(&pdev->dev, mc);  in tegra_mc_probe()
808 		if (IS_ERR(mc->gart)) {  in tegra_mc_probe()
809 			dev_err(&pdev->dev, "failed to probe GART: %ld\n",  in tegra_mc_probe()
810 				PTR_ERR(mc->gart));  in tegra_mc_probe()
811 			mc->gart = NULL;  in tegra_mc_probe()
822 	if (mc->soc->ops && mc->soc->ops->suspend)  in tegra_mc_suspend()
823 		return mc->soc->ops->suspend(mc);  in tegra_mc_suspend()
832 	if (mc->soc->ops && mc->soc->ops->resume)  in tegra_mc_resume()
833 		return mc->soc->ops->resume(mc);  in tegra_mc_resume()
844 		.name = "tegra-mc",