305 static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)  in at_xdmac_chan_reg_base()  argument
307 	return atxdmac->regs + (atxdmac->layout->chan_cc_reg_base + chan_nb * 0x40);  in at_xdmac_chan_reg_base()
310 #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))  argument
311 #define at_xdmac_write(atxdmac, reg, value) \  argument
312 	writel_relaxed((value), (atxdmac)->regs + (reg))
368 static void at_xdmac_off(struct at_xdmac *atxdmac)  in at_xdmac_off()  argument
370 	at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);  in at_xdmac_off()
373 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS))  in at_xdmac_off()
376 	at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);  in at_xdmac_off()
383 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()  local
396 	if (atxdmac->layout->sdif)  in at_xdmac_start_xfer()
452 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
456 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
493 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()  local
496 	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);  in at_xdmac_alloc_desc()
563 	struct at_xdmac		*atxdmac = of_dma->of_dma_data;  in at_xdmac_xlate()  local
566 	struct device		*dev = atxdmac->dma.dev;  in at_xdmac_xlate()
573 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
593 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_compute_chan_conf()  local
605 		if (atxdmac->layout->sdif)  in at_xdmac_compute_chan_conf()
630 		if (atxdmac->layout->sdif)  in at_xdmac_compute_chan_conf()
1454 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()  local
1501 		at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);  in at_xdmac_tx_status()
1559 		at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);  in at_xdmac_tx_status()
1638 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()  local
1657 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1658 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1728 	struct at_xdmac		*atxdmac = (struct at_xdmac *)dev_id;  in at_xdmac_interrupt()  local
1735 		imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in at_xdmac_interrupt()
1736 		status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);  in at_xdmac_interrupt()
1739 		dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1747 		for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_interrupt()
1751 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1755 			dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1769 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1815 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()  local
1824 	at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask);  in at_xdmac_device_pause()
1836 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()  local
1847 	at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask);  in at_xdmac_device_resume()
1858 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()  local
1864 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1865 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1917 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()  local
1923 		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);  in at_xdmac_free_chan_resources()
1932 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_prepare()  local
1935 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1951 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_suspend()  local
1954 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1966 	atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in atmel_xdmac_suspend()
1968 	at_xdmac_off(atxdmac);  in atmel_xdmac_suspend()
1969 	clk_disable_unprepare(atxdmac->clk);  in atmel_xdmac_suspend()
1975 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_resume()  local
1981 	ret = clk_prepare_enable(atxdmac->clk);  in atmel_xdmac_resume()
1986 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in atmel_xdmac_resume()
1987 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1992 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);  in atmel_xdmac_resume()
1993 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
2003 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
2012 	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);  in at_xdmac_axi_config()  local
2016 	if (!atxdmac->layout->axi_config)  in at_xdmac_axi_config()
2026 		at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_M2M);  in at_xdmac_axi_config()
2027 		at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_M2M);  in at_xdmac_axi_config()
2029 		at_xdmac_write(atxdmac, AT_XDMAC_GCFG, AT_XDMAC_GCFG_P2M);  in at_xdmac_axi_config()
2030 		at_xdmac_write(atxdmac, AT_XDMAC_GWAC, AT_XDMAC_GWAC_P2M);  in at_xdmac_axi_config()
2036 	struct at_xdmac	*atxdmac;  in at_xdmac_probe()  local
2062 	size = sizeof(*atxdmac);  in at_xdmac_probe()
2064 	atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in at_xdmac_probe()
2065 	if (!atxdmac) {  in at_xdmac_probe()
2070 	atxdmac->regs = base;  in at_xdmac_probe()
2071 	atxdmac->irq = irq;  in at_xdmac_probe()
2073 	atxdmac->layout = of_device_get_match_data(&pdev->dev);  in at_xdmac_probe()
2074 	if (!atxdmac->layout)  in at_xdmac_probe()
2077 	atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");  in at_xdmac_probe()
2078 	if (IS_ERR(atxdmac->clk)) {  in at_xdmac_probe()
2080 		return PTR_ERR(atxdmac->clk);  in at_xdmac_probe()
2084 	ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);  in at_xdmac_probe()
2090 	ret = clk_prepare_enable(atxdmac->clk);  in at_xdmac_probe()
2096 	atxdmac->at_xdmac_desc_pool =  in at_xdmac_probe()
2099 	if (!atxdmac->at_xdmac_desc_pool) {  in at_xdmac_probe()
2105 	dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2106 	dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2107 	dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2108 	dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2109 	dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2110 	dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2115 	dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2116 	atxdmac->dma.dev				= &pdev->dev;  in at_xdmac_probe()
2117 	atxdmac->dma.device_alloc_chan_resources	= at_xdmac_alloc_chan_resources;  in at_xdmac_probe()
2118 	atxdmac->dma.device_free_chan_resources		= at_xdmac_free_chan_resources;  in at_xdmac_probe()
2119 	atxdmac->dma.device_tx_status			= at_xdmac_tx_status;  in at_xdmac_probe()
2120 	atxdmac->dma.device_issue_pending		= at_xdmac_issue_pending;  in at_xdmac_probe()
2121 	atxdmac->dma.device_prep_dma_cyclic		= at_xdmac_prep_dma_cyclic;  in at_xdmac_probe()
2122 	atxdmac->dma.device_prep_interleaved_dma	= at_xdmac_prep_interleaved;  in at_xdmac_probe()
2123 	atxdmac->dma.device_prep_dma_memcpy		= at_xdmac_prep_dma_memcpy;  in at_xdmac_probe()
2124 	atxdmac->dma.device_prep_dma_memset		= at_xdmac_prep_dma_memset;  in at_xdmac_probe()
2125 	atxdmac->dma.device_prep_dma_memset_sg		= at_xdmac_prep_dma_memset_sg;  in at_xdmac_probe()
2126 	atxdmac->dma.device_prep_slave_sg		= at_xdmac_prep_slave_sg;  in at_xdmac_probe()
2127 	atxdmac->dma.device_config			= at_xdmac_device_config;  in at_xdmac_probe()
2128 	atxdmac->dma.device_pause			= at_xdmac_device_pause;  in at_xdmac_probe()
2129 	atxdmac->dma.device_resume			= at_xdmac_device_resume;  in at_xdmac_probe()
2130 	atxdmac->dma.device_terminate_all		= at_xdmac_device_terminate_all;  in at_xdmac_probe()
2131 	atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2132 	atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2133 	atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_xdmac_probe()
2134 	atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_xdmac_probe()
2137 	at_xdmac_off(atxdmac);  in at_xdmac_probe()
2140 	INIT_LIST_HEAD(&atxdmac->dma.channels);  in at_xdmac_probe()
2142 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
2144 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2146 			      &atxdmac->dma.channels);  in at_xdmac_probe()
2148 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2160 	platform_set_drvdata(pdev, atxdmac);  in at_xdmac_probe()
2162 	ret = dma_async_device_register(&atxdmac->dma);  in at_xdmac_probe()
2169 					 at_xdmac_xlate, atxdmac);  in at_xdmac_probe()
2176 		 nr_channels, atxdmac->regs);  in at_xdmac_probe()
2183 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_probe()
2185 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_probe()
2187 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_probe()
2193 	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);  in at_xdmac_remove()  local
2196 	at_xdmac_off(atxdmac);  in at_xdmac_remove()
2198 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_remove()
2199 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_remove()
2201 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_remove()
2203 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_remove()
2204 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()