144 	struct imxdma_engine		*imxdma;  member
190 	struct imxdma_engine	*imxdma;  member
207 static inline int is_imx1_dma(struct imxdma_engine *imxdma)  in is_imx1_dma()  argument
209 	return imxdma->devtype == IMX1_DMA;  in is_imx1_dma()
212 static inline int is_imx27_dma(struct imxdma_engine *imxdma)  in is_imx27_dma()  argument
214 	return imxdma->devtype == IMX27_DMA;  in is_imx27_dma()
237 static void imx_dmav1_writel(struct imxdma_engine *imxdma, unsigned val,  in imx_dmav1_writel()  argument
240 	__raw_writel(val, imxdma->base + offset);  in imx_dmav1_writel()
243 static unsigned imx_dmav1_readl(struct imxdma_engine *imxdma, unsigned offset)  in imx_dmav1_readl()  argument
245 	return __raw_readl(imxdma->base + offset);  in imx_dmav1_readl()
250 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_hw_chain()  local
252 	if (is_imx27_dma(imxdma))  in imxdma_hw_chain()
264 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_sg_next()  local
273 		imx_dmav1_writel(imxdma, sg->dma_address,  in imxdma_sg_next()
276 		imx_dmav1_writel(imxdma, sg->dma_address,  in imxdma_sg_next()
279 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));  in imxdma_sg_next()
281 	dev_dbg(imxdma->dev, " %s channel: %d dst 0x%08x, src 0x%08x, "  in imxdma_sg_next()
283 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),  in imxdma_sg_next()
284 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),  in imxdma_sg_next()
285 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));  in imxdma_sg_next()
291 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_enable_hw()  local
295 	dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);  in imxdma_enable_hw()
299 	imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);  in imxdma_enable_hw()
300 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) &  in imxdma_enable_hw()
302 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) |  in imxdma_enable_hw()
305 	if (!is_imx1_dma(imxdma) &&  in imxdma_enable_hw()
311 			tmp = imx_dmav1_readl(imxdma, DMA_CCR(channel));  in imxdma_enable_hw()
312 			imx_dmav1_writel(imxdma, tmp | CCR_RPT | CCR_ACRPT,  in imxdma_enable_hw()
322 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_disable_hw()  local
326 	dev_dbg(imxdma->dev, "%s channel %d\n", __func__, channel);  in imxdma_disable_hw()
332 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_DIMR) |  in imxdma_disable_hw()
334 	imx_dmav1_writel(imxdma, imx_dmav1_readl(imxdma, DMA_CCR(channel)) &  in imxdma_disable_hw()
336 	imx_dmav1_writel(imxdma, 1 << channel, DMA_DISR);  in imxdma_disable_hw()
343 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_watchdog()  local
346 	imx_dmav1_writel(imxdma, 0, DMA_CCR(channel));  in imxdma_watchdog()
350 	dev_dbg(imxdma->dev, "channel %d: watchdog timeout!\n",  in imxdma_watchdog()
356 	struct imxdma_engine *imxdma = dev_id;  in imxdma_err_handler()  local
361 	disr = imx_dmav1_readl(imxdma, DMA_DISR);  in imxdma_err_handler()
363 	err_mask = imx_dmav1_readl(imxdma, DMA_DBTOSR) |  in imxdma_err_handler()
364 		   imx_dmav1_readl(imxdma, DMA_DRTOSR) |  in imxdma_err_handler()
365 		   imx_dmav1_readl(imxdma, DMA_DSESR)  |  in imxdma_err_handler()
366 		   imx_dmav1_readl(imxdma, DMA_DBOSR);  in imxdma_err_handler()
371 	imx_dmav1_writel(imxdma, disr & err_mask, DMA_DISR);  in imxdma_err_handler()
378 		if (imx_dmav1_readl(imxdma, DMA_DBTOSR) & (1 << i)) {  in imxdma_err_handler()
379 			imx_dmav1_writel(imxdma, 1 << i, DMA_DBTOSR);  in imxdma_err_handler()
382 		if (imx_dmav1_readl(imxdma, DMA_DRTOSR) & (1 << i)) {  in imxdma_err_handler()
383 			imx_dmav1_writel(imxdma, 1 << i, DMA_DRTOSR);  in imxdma_err_handler()
386 		if (imx_dmav1_readl(imxdma, DMA_DSESR) & (1 << i)) {  in imxdma_err_handler()
387 			imx_dmav1_writel(imxdma, 1 << i, DMA_DSESR);  in imxdma_err_handler()
390 		if (imx_dmav1_readl(imxdma, DMA_DBOSR) & (1 << i)) {  in imxdma_err_handler()
391 			imx_dmav1_writel(imxdma, 1 << i, DMA_DBOSR);  in imxdma_err_handler()
395 		tasklet_schedule(&imxdma->channel[i].dma_tasklet);  in imxdma_err_handler()
397 		dev_warn(imxdma->dev,  in imxdma_err_handler()
409 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in dma_irq_handle_channel()  local
414 	spin_lock_irqsave(&imxdma->lock, flags);  in dma_irq_handle_channel()
416 		spin_unlock_irqrestore(&imxdma->lock, flags);  in dma_irq_handle_channel()
423 	spin_unlock_irqrestore(&imxdma->lock, flags);  in dma_irq_handle_channel()
432 			tmp = imx_dmav1_readl(imxdma, DMA_CCR(chno));  in dma_irq_handle_channel()
442 				imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));  in dma_irq_handle_channel()
444 				imx_dmav1_writel(imxdma, tmp & ~CCR_CEN,  in dma_irq_handle_channel()
449 			imx_dmav1_writel(imxdma, tmp, DMA_CCR(chno));  in dma_irq_handle_channel()
465 	imx_dmav1_writel(imxdma, 0, DMA_CCR(chno));  in dma_irq_handle_channel()
472 	struct imxdma_engine *imxdma = dev_id;  in dma_irq_handler()  local
475 	if (!is_imx1_dma(imxdma))  in dma_irq_handler()
478 	disr = imx_dmav1_readl(imxdma, DMA_DISR);  in dma_irq_handler()
480 	dev_dbg(imxdma->dev, "%s called, disr=0x%08x\n", __func__, disr);  in dma_irq_handler()
482 	imx_dmav1_writel(imxdma, disr, DMA_DISR);  in dma_irq_handler()
485 			dma_irq_handle_channel(&imxdma->channel[i]);  in dma_irq_handler()
494 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_xfer_desc()  local
503 			if ((imxdma->slots_2d[i].count > 0) &&  in imxdma_xfer_desc()
504 			((imxdma->slots_2d[i].xsr != d->x) ||  in imxdma_xfer_desc()
505 			(imxdma->slots_2d[i].ysr != d->y) ||  in imxdma_xfer_desc()
506 			(imxdma->slots_2d[i].wsr != d->w)))  in imxdma_xfer_desc()
514 		imxdma->slots_2d[slot].xsr = d->x;  in imxdma_xfer_desc()
515 		imxdma->slots_2d[slot].ysr = d->y;  in imxdma_xfer_desc()
516 		imxdma->slots_2d[slot].wsr = d->w;  in imxdma_xfer_desc()
517 		imxdma->slots_2d[slot].count++;  in imxdma_xfer_desc()
525 			imx_dmav1_writel(imxdma, d->x, DMA_XSRA);  in imxdma_xfer_desc()
526 			imx_dmav1_writel(imxdma, d->y, DMA_YSRA);  in imxdma_xfer_desc()
527 			imx_dmav1_writel(imxdma, d->w, DMA_WSRA);  in imxdma_xfer_desc()
531 			imx_dmav1_writel(imxdma, d->x, DMA_XSRB);  in imxdma_xfer_desc()
532 			imx_dmav1_writel(imxdma, d->y, DMA_YSRB);  in imxdma_xfer_desc()
533 			imx_dmav1_writel(imxdma, d->w, DMA_WSRB);  in imxdma_xfer_desc()
541 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
542 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
543 		imx_dmav1_writel(imxdma, d->config_mem | (d->config_port << 2),  in imxdma_xfer_desc()
546 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));  in imxdma_xfer_desc()
548 		dev_dbg(imxdma->dev,  in imxdma_xfer_desc()
559 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
561 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,  in imxdma_xfer_desc()
564 			dev_dbg(imxdma->dev,  in imxdma_xfer_desc()
570 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
572 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,  in imxdma_xfer_desc()
575 			dev_dbg(imxdma->dev,  in imxdma_xfer_desc()
581 			dev_err(imxdma->dev, "%s channel: %d bad dma mode\n",  in imxdma_xfer_desc()
599 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tasklet()  local
603 	spin_lock_irqsave(&imxdma->lock, flags);  in imxdma_tasklet()
607 		spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_tasklet()
623 		imxdma->slots_2d[imxdmac->slot_2d].count--;  in imxdma_tasklet()
634 			dev_warn(imxdma->dev, "%s: channel: %d couldn't xfer desc\n",  in imxdma_tasklet()
638 	spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_tasklet()
646 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_terminate_all()  local
651 	spin_lock_irqsave(&imxdma->lock, flags);  in imxdma_terminate_all()
654 	spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_terminate_all()
663 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_config_write()  local
697 	imx_dmav1_writel(imxdma, imxdmac->dma_request,  in imxdma_config_write()
701 	imx_dmav1_writel(imxdma, imxdmac->watermark_level *  in imxdma_config_write()
727 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tx_submit()  local
731 	spin_lock_irqsave(&imxdma->lock, flags);  in imxdma_tx_submit()
734 	spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_tx_submit()
772 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_free_chan_resources()  local
776 	spin_lock_irqsave(&imxdma->lock, flags);  in imxdma_free_chan_resources()
782 	spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_free_chan_resources()
853 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_cyclic()  local
858 	dev_dbg(imxdma->dev, "%s channel: %d buf_len=%zu period_len=%zu\n",  in imxdma_prep_dma_cyclic()
910 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_memcpy()  local
913 	dev_dbg(imxdma->dev, "%s channel: %d src=0x%llx dst=0x%llx len=%zu\n",  in imxdma_prep_dma_memcpy()
941 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_interleaved()  local
944 	dev_dbg(imxdma->dev, "%s channel: %d src_start=0x%llx dst_start=0x%llx\n"  in imxdma_prep_dma_interleaved()
983 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_issue_pending()  local
987 	spin_lock_irqsave(&imxdma->lock, flags);  in imxdma_issue_pending()
994 			dev_warn(imxdma->dev,  in imxdma_issue_pending()
1002 	spin_unlock_irqrestore(&imxdma->lock, flags);  in imxdma_issue_pending()
1010 	if (chan->device->dev != fdata->imxdma->dev)  in imxdma_filter_fn()
1023 	struct imxdma_engine *imxdma = ofdma->of_dma_data;  in imxdma_xlate()  local
1025 		.imxdma = imxdma,  in imxdma_xlate()
1033 	return dma_request_channel(imxdma->dma_device.cap_mask,  in imxdma_xlate()
1039 	struct imxdma_engine *imxdma;  in imxdma_probe()  local
1043 	imxdma = devm_kzalloc(&pdev->dev, sizeof(*imxdma), GFP_KERNEL);  in imxdma_probe()
1044 	if (!imxdma)  in imxdma_probe()
1047 	imxdma->dev = &pdev->dev;  in imxdma_probe()
1048 	imxdma->devtype = (uintptr_t)of_device_get_match_data(&pdev->dev);  in imxdma_probe()
1050 	imxdma->base = devm_platform_ioremap_resource(pdev, 0);  in imxdma_probe()
1051 	if (IS_ERR(imxdma->base))  in imxdma_probe()
1052 		return PTR_ERR(imxdma->base);  in imxdma_probe()
1058 	imxdma->dma_ipg = devm_clk_get(&pdev->dev, "ipg");  in imxdma_probe()
1059 	if (IS_ERR(imxdma->dma_ipg))  in imxdma_probe()
1060 		return PTR_ERR(imxdma->dma_ipg);  in imxdma_probe()
1062 	imxdma->dma_ahb = devm_clk_get(&pdev->dev, "ahb");  in imxdma_probe()
1063 	if (IS_ERR(imxdma->dma_ahb))  in imxdma_probe()
1064 		return PTR_ERR(imxdma->dma_ahb);  in imxdma_probe()
1066 	ret = clk_prepare_enable(imxdma->dma_ipg);  in imxdma_probe()
1069 	ret = clk_prepare_enable(imxdma->dma_ahb);  in imxdma_probe()
1074 	imx_dmav1_writel(imxdma, DCR_DRST, DMA_DCR);  in imxdma_probe()
1076 	if (is_imx1_dma(imxdma)) {  in imxdma_probe()
1078 				       dma_irq_handler, 0, "DMA", imxdma);  in imxdma_probe()
1080 			dev_warn(imxdma->dev, "Can't register IRQ for DMA\n");  in imxdma_probe()
1083 		imxdma->irq = irq;  in imxdma_probe()
1092 				       imxdma_err_handler, 0, "DMA", imxdma);  in imxdma_probe()
1094 			dev_warn(imxdma->dev, "Can't register ERRIRQ for DMA\n");  in imxdma_probe()
1097 		imxdma->irq_err = irq_err;  in imxdma_probe()
1101 	imx_dmav1_writel(imxdma, DCR_DEN, DMA_DCR);  in imxdma_probe()
1104 	imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DISR);  in imxdma_probe()
1107 	imx_dmav1_writel(imxdma, (1 << IMX_DMA_CHANNELS) - 1, DMA_DIMR);  in imxdma_probe()
1109 	INIT_LIST_HEAD(&imxdma->dma_device.channels);  in imxdma_probe()
1111 	dma_cap_set(DMA_SLAVE, imxdma->dma_device.cap_mask);  in imxdma_probe()
1112 	dma_cap_set(DMA_CYCLIC, imxdma->dma_device.cap_mask);  in imxdma_probe()
1113 	dma_cap_set(DMA_MEMCPY, imxdma->dma_device.cap_mask);  in imxdma_probe()
1114 	dma_cap_set(DMA_INTERLEAVE, imxdma->dma_device.cap_mask);  in imxdma_probe()
1118 		imxdma->slots_2d[i].count = 0;  in imxdma_probe()
1120 	spin_lock_init(&imxdma->lock);  in imxdma_probe()
1124 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_probe()
1126 		if (!is_imx1_dma(imxdma)) {  in imxdma_probe()
1128 					dma_irq_handler, 0, "DMA", imxdma);  in imxdma_probe()
1130 				dev_warn(imxdma->dev, "Can't register IRQ %d "  in imxdma_probe()
1140 		imxdmac->imxdma = imxdma;  in imxdma_probe()
1147 		imxdmac->chan.device = &imxdma->dma_device;  in imxdma_probe()
1153 			      &imxdma->dma_device.channels);  in imxdma_probe()
1156 	imxdma->dma_device.dev = &pdev->dev;  in imxdma_probe()
1158 	imxdma->dma_device.device_alloc_chan_resources = imxdma_alloc_chan_resources;  in imxdma_probe()
1159 	imxdma->dma_device.device_free_chan_resources = imxdma_free_chan_resources;  in imxdma_probe()
1160 	imxdma->dma_device.device_tx_status = imxdma_tx_status;  in imxdma_probe()
1161 	imxdma->dma_device.device_prep_slave_sg = imxdma_prep_slave_sg;  in imxdma_probe()
1162 	imxdma->dma_device.device_prep_dma_cyclic = imxdma_prep_dma_cyclic;  in imxdma_probe()
1163 	imxdma->dma_device.device_prep_dma_memcpy = imxdma_prep_dma_memcpy;  in imxdma_probe()
1164 	imxdma->dma_device.device_prep_interleaved_dma = imxdma_prep_dma_interleaved;  in imxdma_probe()
1165 	imxdma->dma_device.device_config = imxdma_config;  in imxdma_probe()
1166 	imxdma->dma_device.device_terminate_all = imxdma_terminate_all;  in imxdma_probe()
1167 	imxdma->dma_device.device_issue_pending = imxdma_issue_pending;  in imxdma_probe()
1169 	platform_set_drvdata(pdev, imxdma);  in imxdma_probe()
1171 	imxdma->dma_device.copy_align = DMAENGINE_ALIGN_4_BYTES;  in imxdma_probe()
1172 	dma_set_max_seg_size(imxdma->dma_device.dev, 0xffffff);  in imxdma_probe()
1174 	ret = dma_async_device_register(&imxdma->dma_device);  in imxdma_probe()
1182 				imxdma_xlate, imxdma);  in imxdma_probe()
1192 	dma_async_device_unregister(&imxdma->dma_device);  in imxdma_probe()
1194 	clk_disable_unprepare(imxdma->dma_ahb);  in imxdma_probe()
1196 	clk_disable_unprepare(imxdma->dma_ipg);  in imxdma_probe()
1200 static void imxdma_free_irq(struct platform_device *pdev, struct imxdma_engine *imxdma)  in imxdma_free_irq()  argument
1204 	if (is_imx1_dma(imxdma)) {  in imxdma_free_irq()
1205 		disable_irq(imxdma->irq);  in imxdma_free_irq()
1206 		disable_irq(imxdma->irq_err);  in imxdma_free_irq()
1210 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_free_irq()
1212 		if (!is_imx1_dma(imxdma))  in imxdma_free_irq()
1221 	struct imxdma_engine *imxdma = platform_get_drvdata(pdev);  in imxdma_remove()  local
1223 	imxdma_free_irq(pdev, imxdma);  in imxdma_remove()
1225         dma_async_device_unregister(&imxdma->dma_device);  in imxdma_remove()
1230 	clk_disable_unprepare(imxdma->dma_ipg);  in imxdma_remove()
1231 	clk_disable_unprepare(imxdma->dma_ahb);  in imxdma_remove()