87 	struct cppi41_dd *cdd;  member
253 static struct cppi41_channel *desc_to_chan(struct cppi41_dd *cdd, u32 desc)  in desc_to_chan()  argument
261 	if (!((desc >= cdd->descs_phys) &&  in desc_to_chan()
262 			(desc < (cdd->descs_phys + descs_size)))) {  in desc_to_chan()
266 	desc_num = (desc - cdd->descs_phys) / sizeof(struct cppi41_desc);  in desc_to_chan()
268 	c = cdd->chan_busy[desc_num];  in desc_to_chan()
269 	cdd->chan_busy[desc_num] = NULL;  in desc_to_chan()
272 	pm_runtime_put(cdd->ddev.dev);  in desc_to_chan()
292 static u32 cppi41_pop_desc(struct cppi41_dd *cdd, unsigned queue_num)  in cppi41_pop_desc()  argument
296 	desc = cppi_readl(cdd->qmgr_mem + QMGR_QUEUE_D(queue_num));  in cppi41_pop_desc()
303 	struct cppi41_dd *cdd = data;  in cppi41_irq()  local
304 	u16 first_completion_queue = cdd->first_completion_queue;  in cppi41_irq()
305 	u16 qmgr_num_pend = cdd->qmgr_num_pend;  in cppi41_irq()
314 		val = cppi_readl(cdd->qmgr_mem + QMGR_PEND(i));  in cppi41_irq()
335 			WARN_ON(cdd->is_suspended);  in cppi41_irq()
340 			desc = cppi41_pop_desc(cdd, q_num);  in cppi41_irq()
341 			c = desc_to_chan(cdd, desc);  in cppi41_irq()
373 	struct cppi41_dd *cdd = c->cdd;  in cppi41_dma_alloc_chan_resources()  local
376 	error = pm_runtime_get_sync(cdd->ddev.dev);  in cppi41_dma_alloc_chan_resources()
378 		dev_err(cdd->ddev.dev, "%s pm runtime get: %i\n",  in cppi41_dma_alloc_chan_resources()
380 		pm_runtime_put_noidle(cdd->ddev.dev);  in cppi41_dma_alloc_chan_resources()
392 	pm_runtime_mark_last_busy(cdd->ddev.dev);  in cppi41_dma_alloc_chan_resources()
393 	pm_runtime_put_autosuspend(cdd->ddev.dev);  in cppi41_dma_alloc_chan_resources()
401 	struct cppi41_dd *cdd = c->cdd;  in cppi41_dma_free_chan_resources()  local
404 	error = pm_runtime_get_sync(cdd->ddev.dev);  in cppi41_dma_free_chan_resources()
406 		pm_runtime_put_noidle(cdd->ddev.dev);  in cppi41_dma_free_chan_resources()
411 	WARN_ON(!list_empty(&cdd->pending));  in cppi41_dma_free_chan_resources()
413 	pm_runtime_mark_last_busy(cdd->ddev.dev);  in cppi41_dma_free_chan_resources()
414 	pm_runtime_put_autosuspend(cdd->ddev.dev);  in cppi41_dma_free_chan_resources()
432 	struct cppi41_dd *cdd = c->cdd;  in push_desc_queue()  local
462 	pm_runtime_get(cdd->ddev.dev);  in push_desc_queue()
465 	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);  in push_desc_queue()
466 	WARN_ON(cdd->chan_busy[desc_num]);  in push_desc_queue()
467 	cdd->chan_busy[desc_num] = c;  in push_desc_queue()
471 	cppi_writel(reg, cdd->qmgr_mem + QMGR_QUEUE_D(c->q_num));  in push_desc_queue()
479 static void cppi41_run_queue(struct cppi41_dd *cdd)  in cppi41_run_queue()  argument
483 	list_for_each_entry_safe(c, _c, &cdd->pending, node) {  in cppi41_run_queue()
492 	struct cppi41_dd *cdd = c->cdd;  in cppi41_dma_issue_pending()  local
496 	error = pm_runtime_get(cdd->ddev.dev);  in cppi41_dma_issue_pending()
498 		pm_runtime_put_noidle(cdd->ddev.dev);  in cppi41_dma_issue_pending()
499 		dev_err(cdd->ddev.dev, "Failed to pm_runtime_get: %i\n",  in cppi41_dma_issue_pending()
505 	spin_lock_irqsave(&cdd->lock, flags);  in cppi41_dma_issue_pending()
506 	list_add_tail(&c->node, &cdd->pending);  in cppi41_dma_issue_pending()
507 	if (!cdd->is_suspended)  in cppi41_dma_issue_pending()
508 		cppi41_run_queue(cdd);  in cppi41_dma_issue_pending()
509 	spin_unlock_irqrestore(&cdd->lock, flags);  in cppi41_dma_issue_pending()
511 	pm_runtime_mark_last_busy(cdd->ddev.dev);  in cppi41_dma_issue_pending()
512 	pm_runtime_put_autosuspend(cdd->ddev.dev);  in cppi41_dma_issue_pending()
624 	struct cppi41_dd *cdd = c->cdd;  in cppi41_tear_down_chan()  local
630 	td = cdd->cd;  in cppi41_tear_down_chan()
631 	td += cdd->first_td_desc;  in cppi41_tear_down_chan()
633 	td_desc_phys = cdd->descs_phys;  in cppi41_tear_down_chan()
634 	td_desc_phys += cdd->first_td_desc * sizeof(struct cppi41_desc);  in cppi41_tear_down_chan()
642 		cppi_writel(reg, cdd->qmgr_mem +  in cppi41_tear_down_chan()
643 				QMGR_QUEUE_D(cdd->td_queue.submit));  in cppi41_tear_down_chan()
649 			reg |= cdd->td_queue.complete;  in cppi41_tear_down_chan()
659 		desc_phys = cppi41_pop_desc(cdd, cdd->td_queue.complete);  in cppi41_tear_down_chan()
661 			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);  in cppi41_tear_down_chan()
697 		desc_phys = cppi41_pop_desc(cdd, c->q_num);  in cppi41_tear_down_chan()
699 			desc_phys = cppi41_pop_desc(cdd, c->q_comp_num);  in cppi41_tear_down_chan()
719 	struct cppi41_dd *cdd = c->cdd;  in cppi41_stop_chan()  local
725 	desc_num = (desc_phys - cdd->descs_phys) / sizeof(struct cppi41_desc);  in cppi41_stop_chan()
726 	if (!cdd->chan_busy[desc_num])  in cppi41_stop_chan()
733 	WARN_ON(!cdd->chan_busy[desc_num]);  in cppi41_stop_chan()
734 	cdd->chan_busy[desc_num] = NULL;  in cppi41_stop_chan()
737 	pm_runtime_put(cdd->ddev.dev);  in cppi41_stop_chan()
742 static int cppi41_add_chans(struct device *dev, struct cppi41_dd *cdd)  in cppi41_add_chans()  argument
746 	u32 n_chans = cdd->n_chans;  in cppi41_add_chans()
761 		cchan->cdd = cdd;  in cppi41_add_chans()
763 			cchan->gcr_reg = cdd->ctrl_mem + DMA_TXGCR(i >> 1);  in cppi41_add_chans()
766 			cchan->gcr_reg = cdd->ctrl_mem + DMA_RXGCR(i >> 1);  in cppi41_add_chans()
770 		cchan->desc = &cdd->cd[i];  in cppi41_add_chans()
771 		cchan->desc_phys = cdd->descs_phys;  in cppi41_add_chans()
773 		cchan->chan.device = &cdd->ddev;  in cppi41_add_chans()
774 		list_add_tail(&cchan->chan.device_node, &cdd->ddev.channels);  in cppi41_add_chans()
776 	cdd->first_td_desc = n_chans;  in cppi41_add_chans()
781 static void purge_descs(struct device *dev, struct cppi41_dd *cdd)  in purge_descs()  argument
790 		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMBASE(i));  in purge_descs()
791 		cppi_writel(0, cdd->qmgr_mem + QMGR_MEMCTRL(i));  in purge_descs()
793 		dma_free_coherent(dev, mem_decs, cdd->cd,  in purge_descs()
794 				cdd->descs_phys);  in purge_descs()
798 static void disable_sched(struct cppi41_dd *cdd)  in disable_sched()  argument
800 	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);  in disable_sched()
803 static void deinit_cppi41(struct device *dev, struct cppi41_dd *cdd)  in deinit_cppi41()  argument
805 	disable_sched(cdd);  in deinit_cppi41()
807 	purge_descs(dev, cdd);  in deinit_cppi41()
809 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in deinit_cppi41()
810 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in deinit_cppi41()
811 	dma_free_coherent(dev, QMGR_SCRATCH_SIZE, cdd->qmgr_scratch,  in deinit_cppi41()
812 			cdd->scratch_phys);  in deinit_cppi41()
815 static int init_descs(struct device *dev, struct cppi41_dd *cdd)  in init_descs()  argument
839 		cdd->cd = dma_alloc_coherent(dev, mem_decs,  in init_descs()
840 				&cdd->descs_phys, GFP_KERNEL);  in init_descs()
841 		if (!cdd->cd)  in init_descs()
844 		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));  in init_descs()
845 		cppi_writel(reg, cdd->qmgr_mem + QMGR_MEMCTRL(i));  in init_descs()
852 static void init_sched(struct cppi41_dd *cdd)  in init_sched()  argument
859 	cppi_writel(0, cdd->sched_mem + DMA_SCHED_CTRL);  in init_sched()
860 	for (ch = 0; ch < cdd->n_chans; ch += 2) {  in init_sched()
867 		cppi_writel(reg, cdd->sched_mem + DMA_SCHED_WORD(word));  in init_sched()
870 	reg = cdd->n_chans * 2 - 1;  in init_sched()
872 	cppi_writel(reg, cdd->sched_mem + DMA_SCHED_CTRL);  in init_sched()
875 static int init_cppi41(struct device *dev, struct cppi41_dd *cdd)  in init_cppi41()  argument
880 	cdd->qmgr_scratch = dma_alloc_coherent(dev, QMGR_SCRATCH_SIZE,  in init_cppi41()
881 			&cdd->scratch_phys, GFP_KERNEL);  in init_cppi41()
882 	if (!cdd->qmgr_scratch)  in init_cppi41()
885 	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in init_cppi41()
886 	cppi_writel(TOTAL_DESCS_NUM, cdd->qmgr_mem + QMGR_LRAM_SIZE);  in init_cppi41()
887 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);  in init_cppi41()
889 	ret = init_descs(dev, cdd);  in init_cppi41()
893 	cppi_writel(cdd->td_queue.submit, cdd->ctrl_mem + DMA_TDFDQ);  in init_cppi41()
894 	init_sched(cdd);  in init_cppi41()
898 	deinit_cppi41(dev, cdd);  in init_cppi41()
915 	struct cppi41_dd *cdd;  in cpp41_dma_filter_fn()  local
929 	cdd = cchan->cdd;  in cpp41_dma_filter_fn()
931 		queues = cdd->queues_tx;  in cpp41_dma_filter_fn()
933 		queues = cdd->queues_rx;  in cpp41_dma_filter_fn()
1005 	struct cppi41_dd *cdd;  in cppi41_dma_probe()  local
1017 	cdd = devm_kzalloc(&pdev->dev, sizeof(*cdd), GFP_KERNEL);  in cppi41_dma_probe()
1018 	if (!cdd)  in cppi41_dma_probe()
1021 	dma_cap_set(DMA_SLAVE, cdd->ddev.cap_mask);  in cppi41_dma_probe()
1022 	cdd->ddev.device_alloc_chan_resources = cppi41_dma_alloc_chan_resources;  in cppi41_dma_probe()
1023 	cdd->ddev.device_free_chan_resources = cppi41_dma_free_chan_resources;  in cppi41_dma_probe()
1024 	cdd->ddev.device_tx_status = cppi41_dma_tx_status;  in cppi41_dma_probe()
1025 	cdd->ddev.device_issue_pending = cppi41_dma_issue_pending;  in cppi41_dma_probe()
1026 	cdd->ddev.device_prep_slave_sg = cppi41_dma_prep_slave_sg;  in cppi41_dma_probe()
1027 	cdd->ddev.device_terminate_all = cppi41_stop_chan;  in cppi41_dma_probe()
1028 	cdd->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in cppi41_dma_probe()
1029 	cdd->ddev.src_addr_widths = CPPI41_DMA_BUSWIDTHS;  in cppi41_dma_probe()
1030 	cdd->ddev.dst_addr_widths = CPPI41_DMA_BUSWIDTHS;  in cppi41_dma_probe()
1031 	cdd->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in cppi41_dma_probe()
1032 	cdd->ddev.dev = dev;  in cppi41_dma_probe()
1033 	INIT_LIST_HEAD(&cdd->ddev.channels);  in cppi41_dma_probe()
1034 	cpp41_dma_info.dma_cap = cdd->ddev.cap_mask;  in cppi41_dma_probe()
1042 	cdd->ctrl_mem = devm_ioremap_resource(dev, mem);  in cppi41_dma_probe()
1043 	if (IS_ERR(cdd->ctrl_mem))  in cppi41_dma_probe()
1044 		return PTR_ERR(cdd->ctrl_mem);  in cppi41_dma_probe()
1047 	cdd->sched_mem = devm_ioremap_resource(dev, mem);  in cppi41_dma_probe()
1048 	if (IS_ERR(cdd->sched_mem))  in cppi41_dma_probe()
1049 		return PTR_ERR(cdd->sched_mem);  in cppi41_dma_probe()
1052 	cdd->qmgr_mem = devm_ioremap_resource(dev, mem);  in cppi41_dma_probe()
1053 	if (IS_ERR(cdd->qmgr_mem))  in cppi41_dma_probe()
1054 		return PTR_ERR(cdd->qmgr_mem);  in cppi41_dma_probe()
1056 	spin_lock_init(&cdd->lock);  in cppi41_dma_probe()
1057 	INIT_LIST_HEAD(&cdd->pending);  in cppi41_dma_probe()
1059 	platform_set_drvdata(pdev, cdd);  in cppi41_dma_probe()
1068 	cdd->queues_rx = glue_info->queues_rx;  in cppi41_dma_probe()
1069 	cdd->queues_tx = glue_info->queues_tx;  in cppi41_dma_probe()
1070 	cdd->td_queue = glue_info->td_queue;  in cppi41_dma_probe()
1071 	cdd->qmgr_num_pend = glue_info->qmgr_num_pend;  in cppi41_dma_probe()
1072 	cdd->first_completion_queue = glue_info->first_completion_queue;  in cppi41_dma_probe()
1075 				   "#dma-channels", &cdd->n_chans);  in cppi41_dma_probe()
1079 	ret = init_cppi41(dev, cdd);  in cppi41_dma_probe()
1083 	ret = cppi41_add_chans(dev, cdd);  in cppi41_dma_probe()
1094 			dev_name(dev), cdd);  in cppi41_dma_probe()
1097 	cdd->irq = irq;  in cppi41_dma_probe()
1099 	ret = dma_async_device_register(&cdd->ddev);  in cppi41_dma_probe()
1113 	dma_async_device_unregister(&cdd->ddev);  in cppi41_dma_probe()
1115 	deinit_cppi41(dev, cdd);  in cppi41_dma_probe()
1127 	struct cppi41_dd *cdd = platform_get_drvdata(pdev);  in cppi41_dma_remove()  local
1135 	dma_async_device_unregister(&cdd->ddev);  in cppi41_dma_remove()
1137 	devm_free_irq(&pdev->dev, cdd->irq, cdd);  in cppi41_dma_remove()
1138 	deinit_cppi41(&pdev->dev, cdd);  in cppi41_dma_remove()
1147 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_suspend()  local
1149 	cdd->dma_tdfdq = cppi_readl(cdd->ctrl_mem + DMA_TDFDQ);  in cppi41_suspend()
1150 	disable_sched(cdd);  in cppi41_suspend()
1157 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_resume()  local
1162 		cppi_writel(cdd->descs_phys, cdd->qmgr_mem + QMGR_MEMBASE(i));  in cppi41_resume()
1164 	list_for_each_entry(c, &cdd->ddev.channels, chan.device_node)  in cppi41_resume()
1168 	init_sched(cdd);  in cppi41_resume()
1170 	cppi_writel(cdd->dma_tdfdq, cdd->ctrl_mem + DMA_TDFDQ);  in cppi41_resume()
1171 	cppi_writel(cdd->scratch_phys, cdd->qmgr_mem + QMGR_LRAM0_BASE);  in cppi41_resume()
1172 	cppi_writel(QMGR_SCRATCH_SIZE, cdd->qmgr_mem + QMGR_LRAM_SIZE);  in cppi41_resume()
1173 	cppi_writel(0, cdd->qmgr_mem + QMGR_LRAM1_BASE);  in cppi41_resume()
1180 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_runtime_suspend()  local
1183 	spin_lock_irqsave(&cdd->lock, flags);  in cppi41_runtime_suspend()
1184 	cdd->is_suspended = true;  in cppi41_runtime_suspend()
1185 	WARN_ON(!list_empty(&cdd->pending));  in cppi41_runtime_suspend()
1186 	spin_unlock_irqrestore(&cdd->lock, flags);  in cppi41_runtime_suspend()
1193 	struct cppi41_dd *cdd = dev_get_drvdata(dev);  in cppi41_runtime_resume()  local
1196 	spin_lock_irqsave(&cdd->lock, flags);  in cppi41_runtime_resume()
1197 	cdd->is_suspended = false;  in cppi41_runtime_resume()
1198 	cppi41_run_queue(cdd);  in cppi41_runtime_resume()
1199 	spin_unlock_irqrestore(&cdd->lock, flags);  in cppi41_runtime_resume()