419 	struct xilinx_dma_device *xdev;  member
529 	readl_poll_timeout_atomic(chan->xdev->regs + chan->ctrl_offset + reg, \
535 	return ioread32(chan->xdev->regs + reg);  in dma_read()
540 	iowrite32(value, chan->xdev->regs + reg);  in dma_write()
587 	writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg);  in vdma_desc_write_64()
590 	writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4);  in vdma_desc_write_64()
595 	lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg);  in dma_writeq()
861 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_free_tx_descriptor()
866 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_free_tx_descriptor()
872 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_free_tx_descriptor()
937 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_free_chan_resources()
952 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_free_chan_resources()
963 	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA &&  in xilinx_dma_free_chan_resources()
964 	    chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_free_chan_resources()
991 		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_get_residue()
997 				   chan->xdev->max_buffer_len;  in xilinx_dma_get_residue()
998 		} else if (chan->xdev->dma_config->dmatype ==  in xilinx_dma_get_residue()
1005 				   chan->xdev->max_buffer_len;  in xilinx_dma_get_residue()
1014 				chan->xdev->max_buffer_len;  in xilinx_dma_get_residue()
1124 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_alloc_chan_resources()
1167 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_alloc_chan_resources()
1191 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_alloc_chan_resources()
1206 	    ((chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) &&  in xilinx_dma_alloc_chan_resources()
1207 		chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIMCDMA)) {  in xilinx_dma_alloc_chan_resources()
1216 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_alloc_chan_resources()
1224 	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)  in xilinx_dma_alloc_chan_resources()
1245 		     chan->xdev->max_buffer_len);  in xilinx_dma_calc_copysize()
1248 	    chan->xdev->common.copy_align) {  in xilinx_dma_calc_copysize()
1254 				 (1 << chan->xdev->common.copy_align));  in xilinx_dma_calc_copysize()
1289 		if (chan->has_sg && chan->xdev->dma_config->dmatype != XDMA_TYPE_VDMA)  in xilinx_dma_tx_status()
1522 				hw->control & chan->xdev->max_buffer_len);  in xilinx_cdma_start_transfer()
1599 			       hw->control & chan->xdev->max_buffer_len);  in xilinx_dma_start_transfer()
1719 		if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_complete_descriptor()
1727 		if (chan->has_sg && chan->xdev->dma_config->dmatype !=  in xilinx_dma_complete_descriptor()
1821 		chan_offset = chan->xdev->dma_config->max_channels / 2;  in xilinx_mcdma_irq_handler()
1824 	chan = chan->xdev->chan[chan_offset];  in xilinx_mcdma_irq_handler()
1944 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in append_desc_queue()
1949 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in append_desc_queue()
1954 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in append_desc_queue()
1975 	if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA)  in append_desc_queue()
2129 	if (!len || len > chan->xdev->max_buffer_len)  in xilinx_cdma_prep_memcpy()
2258 	if (chan->xdev->has_axistream_connected)  in xilinx_dma_prep_slave_sg()
2430 				     chan->xdev->max_buffer_len);  in xilinx_mcdma_prep_slave_sg()
2508 	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)  in xilinx_dma_terminate_all()
2784 static void xdma_disable_allclks(struct xilinx_dma_device *xdev)  in xdma_disable_allclks()  argument
2786 	clk_disable_unprepare(xdev->rxs_clk);  in xdma_disable_allclks()
2787 	clk_disable_unprepare(xdev->rx_clk);  in xdma_disable_allclks()
2788 	clk_disable_unprepare(xdev->txs_clk);  in xdma_disable_allclks()
2789 	clk_disable_unprepare(xdev->tx_clk);  in xdma_disable_allclks()
2790 	clk_disable_unprepare(xdev->axi_clk);  in xdma_disable_allclks()
2803 static int xilinx_dma_chan_probe(struct xilinx_dma_device *xdev,  in xilinx_dma_chan_probe()  argument
2812 	chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL);  in xilinx_dma_chan_probe()
2816 	chan->dev = xdev->dev;  in xilinx_dma_chan_probe()
2817 	chan->xdev = xdev;  in xilinx_dma_chan_probe()
2819 	chan->ext_addr = xdev->ext_addr;  in xilinx_dma_chan_probe()
2842 		dev_err(xdev->dev, "missing xlnx,datawidth property\n");  in xilinx_dma_chan_probe()
2852 		xdev->common.copy_align = (enum dmaengine_alignment)fls(width - 1);  in xilinx_dma_chan_probe()
2858 		chan->id = xdev->mm2s_chan_id++;  in xilinx_dma_chan_probe()
2862 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_chan_probe()
2866 			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||  in xilinx_dma_chan_probe()
2867 			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_MM2S)  in xilinx_dma_chan_probe()
2875 		chan->id = xdev->s2mm_chan_id++;  in xilinx_dma_chan_probe()
2876 		chan->tdest = chan->id - xdev->dma_config->max_channels / 2;  in xilinx_dma_chan_probe()
2885 		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)  in xilinx_dma_chan_probe()
2890 		if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_chan_probe()
2894 			if (xdev->flush_on_fsync == XILINX_DMA_FLUSH_BOTH ||  in xilinx_dma_chan_probe()
2895 			    xdev->flush_on_fsync == XILINX_DMA_FLUSH_S2MM)  in xilinx_dma_chan_probe()
2899 		dev_err(xdev->dev, "Invalid channel compatible node\n");  in xilinx_dma_chan_probe()
2906 		return dev_err_probe(xdev->dev, chan->irq, "failed to get irq\n");  in xilinx_dma_chan_probe()
2907 	err = request_irq(chan->irq, xdev->dma_config->irq_handler,  in xilinx_dma_chan_probe()
2910 		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);  in xilinx_dma_chan_probe()
2914 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_chan_probe()
2917 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_chan_probe()
2920 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_chan_probe()
2929 	if (xdev->dma_config->dmatype != XDMA_TYPE_VDMA) {  in xilinx_dma_chan_probe()
2930 		if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA ||  in xilinx_dma_chan_probe()
2945 	chan->common.device = &xdev->common;  in xilinx_dma_chan_probe()
2947 	list_add_tail(&chan->common.device_node, &xdev->common.channels);  in xilinx_dma_chan_probe()
2948 	xdev->chan[chan->id] = chan;  in xilinx_dma_chan_probe()
2953 		dev_err(xdev->dev, "Reset channel failed\n");  in xilinx_dma_chan_probe()
2970 static int xilinx_dma_child_probe(struct xilinx_dma_device *xdev,  in xilinx_dma_child_probe()  argument
2977 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA && ret < 0)  in xilinx_dma_child_probe()
2978 		dev_warn(xdev->dev, "missing dma-channels property\n");  in xilinx_dma_child_probe()
2981 		ret = xilinx_dma_chan_probe(xdev, node);  in xilinx_dma_child_probe()
2999 	struct xilinx_dma_device *xdev = ofdma->of_dma_data;  in of_dma_xilinx_xlate()  local
3002 	if (chan_id >= xdev->dma_config->max_channels || !xdev->chan[chan_id])  in of_dma_xilinx_xlate()
3005 	return dma_get_slave_channel(&xdev->chan[chan_id]->common);  in of_dma_xilinx_xlate()
3056 	struct xilinx_dma_device *xdev;  in xilinx_dma_probe()  local
3062 	xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL);  in xilinx_dma_probe()
3063 	if (!xdev)  in xilinx_dma_probe()
3066 	xdev->dev = &pdev->dev;  in xilinx_dma_probe()
3072 			xdev->dma_config = match->data;  in xilinx_dma_probe()
3073 			clk_init = xdev->dma_config->clk_init;  in xilinx_dma_probe()
3077 	err = clk_init(pdev, &xdev->axi_clk, &xdev->tx_clk, &xdev->txs_clk,  in xilinx_dma_probe()
3078 		       &xdev->rx_clk, &xdev->rxs_clk);  in xilinx_dma_probe()
3083 	xdev->regs = devm_platform_ioremap_resource(pdev, 0);  in xilinx_dma_probe()
3084 	if (IS_ERR(xdev->regs)) {  in xilinx_dma_probe()
3085 		err = PTR_ERR(xdev->regs);  in xilinx_dma_probe()
3089 	xdev->max_buffer_len = GENMASK(XILINX_DMA_MAX_TRANS_LEN_MAX - 1, 0);  in xilinx_dma_probe()
3090 	xdev->s2mm_chan_id = xdev->dma_config->max_channels / 2;  in xilinx_dma_probe()
3092 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA ||  in xilinx_dma_probe()
3093 	    xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_probe()
3098 				dev_warn(xdev->dev,  in xilinx_dma_probe()
3102 					dev_warn(xdev->dev, "Please ensure that IP supports buffer length > 23 bits\n");  in xilinx_dma_probe()
3103 				xdev->max_buffer_len =  in xilinx_dma_probe()
3109 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_probe()
3110 		xdev->has_axistream_connected =  in xilinx_dma_probe()
3114 	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_probe()
3118 			dev_err(xdev->dev,  in xilinx_dma_probe()
3124 					   &xdev->flush_on_fsync);  in xilinx_dma_probe()
3126 			dev_warn(xdev->dev,  in xilinx_dma_probe()
3132 		dev_warn(xdev->dev, "missing xlnx,addrwidth property\n");  in xilinx_dma_probe()
3135 		xdev->ext_addr = true;  in xilinx_dma_probe()
3137 		xdev->ext_addr = false;  in xilinx_dma_probe()
3140 	if (xdev->has_axistream_connected)  in xilinx_dma_probe()
3141 		xdev->common.desc_metadata_modes = DESC_METADATA_ENGINE;  in xilinx_dma_probe()
3144 	err = dma_set_mask_and_coherent(xdev->dev, DMA_BIT_MASK(addr_width));  in xilinx_dma_probe()
3146 		dev_err(xdev->dev, "DMA mask error %d\n", err);  in xilinx_dma_probe()
3151 	xdev->common.dev = &pdev->dev;  in xilinx_dma_probe()
3153 	INIT_LIST_HEAD(&xdev->common.channels);  in xilinx_dma_probe()
3154 	if (!(xdev->dma_config->dmatype == XDMA_TYPE_CDMA)) {  in xilinx_dma_probe()
3155 		dma_cap_set(DMA_SLAVE, xdev->common.cap_mask);  in xilinx_dma_probe()
3156 		dma_cap_set(DMA_PRIVATE, xdev->common.cap_mask);  in xilinx_dma_probe()
3159 	xdev->common.device_alloc_chan_resources =  in xilinx_dma_probe()
3161 	xdev->common.device_free_chan_resources =  in xilinx_dma_probe()
3163 	xdev->common.device_terminate_all = xilinx_dma_terminate_all;  in xilinx_dma_probe()
3164 	xdev->common.device_synchronize = xilinx_dma_synchronize;  in xilinx_dma_probe()
3165 	xdev->common.device_tx_status = xilinx_dma_tx_status;  in xilinx_dma_probe()
3166 	xdev->common.device_issue_pending = xilinx_dma_issue_pending;  in xilinx_dma_probe()
3167 	xdev->common.device_config = xilinx_dma_device_config;  in xilinx_dma_probe()
3168 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_probe()
3169 		dma_cap_set(DMA_CYCLIC, xdev->common.cap_mask);  in xilinx_dma_probe()
3170 		xdev->common.device_prep_slave_sg = xilinx_dma_prep_slave_sg;  in xilinx_dma_probe()
3171 		xdev->common.device_prep_dma_cyclic =  in xilinx_dma_probe()
3174 		xdev->common.residue_granularity =  in xilinx_dma_probe()
3176 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_probe()
3177 		dma_cap_set(DMA_MEMCPY, xdev->common.cap_mask);  in xilinx_dma_probe()
3178 		xdev->common.device_prep_dma_memcpy = xilinx_cdma_prep_memcpy;  in xilinx_dma_probe()
3180 		xdev->common.residue_granularity =  in xilinx_dma_probe()
3182 	} else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA) {  in xilinx_dma_probe()
3183 		xdev->common.device_prep_slave_sg = xilinx_mcdma_prep_slave_sg;  in xilinx_dma_probe()
3185 		xdev->common.device_prep_interleaved_dma =  in xilinx_dma_probe()
3189 	platform_set_drvdata(pdev, xdev);  in xilinx_dma_probe()
3193 		err = xilinx_dma_child_probe(xdev, child);  in xilinx_dma_probe()
3200 	if (xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_probe()
3201 		for (i = 0; i < xdev->dma_config->max_channels; i++)  in xilinx_dma_probe()
3202 			if (xdev->chan[i])  in xilinx_dma_probe()
3203 				xdev->chan[i]->num_frms = num_frames;  in xilinx_dma_probe()
3207 	err = dma_async_device_register(&xdev->common);  in xilinx_dma_probe()
3209 		dev_err(xdev->dev, "failed to register the dma device\n");  in xilinx_dma_probe()
3214 					 xdev);  in xilinx_dma_probe()
3217 		dma_async_device_unregister(&xdev->common);  in xilinx_dma_probe()
3221 	if (xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA)  in xilinx_dma_probe()
3223 	else if (xdev->dma_config->dmatype == XDMA_TYPE_CDMA)  in xilinx_dma_probe()
3225 	else if (xdev->dma_config->dmatype == XDMA_TYPE_AXIMCDMA)  in xilinx_dma_probe()
3233 	for (i = 0; i < xdev->dma_config->max_channels; i++)  in xilinx_dma_probe()
3234 		if (xdev->chan[i])  in xilinx_dma_probe()
3235 			xilinx_dma_chan_remove(xdev->chan[i]);  in xilinx_dma_probe()
3237 	xdma_disable_allclks(xdev);  in xilinx_dma_probe()
3250 	struct xilinx_dma_device *xdev = platform_get_drvdata(pdev);  in xilinx_dma_remove()  local
3255 	dma_async_device_unregister(&xdev->common);  in xilinx_dma_remove()
3257 	for (i = 0; i < xdev->dma_config->max_channels; i++)  in xilinx_dma_remove()
3258 		if (xdev->chan[i])  in xilinx_dma_remove()
3259 			xilinx_dma_chan_remove(xdev->chan[i]);  in xilinx_dma_remove()
3261 	xdma_disable_allclks(xdev);  in xilinx_dma_remove()