247 static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)  in at_xdmac_chan_reg_base()  argument
249 	return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);  in at_xdmac_chan_reg_base()
252 #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))  argument
253 #define at_xdmac_write(atxdmac, reg, value) \  argument
254 	writel_relaxed((value), (atxdmac)->regs + (reg))
321 static void at_xdmac_off(struct at_xdmac *atxdmac)  in at_xdmac_off()  argument
323 	at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);  in at_xdmac_off()
326 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS))  in at_xdmac_off()
329 	at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);  in at_xdmac_off()
336 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()  local
403 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
407 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
444 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()  local
447 	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);  in at_xdmac_alloc_desc()
514 	struct at_xdmac		*atxdmac = of_dma->of_dma_data;  in at_xdmac_xlate()  local
517 	struct device		*dev = atxdmac->dma.dev;  in at_xdmac_xlate()
524 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
1394 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()  local
1441 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1499 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1578 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()  local
1597 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1598 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1668 	struct at_xdmac		*atxdmac = (struct at_xdmac *)dev_id;  in at_xdmac_interrupt()  local
1675 		imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in at_xdmac_interrupt()
1676 		status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);  in at_xdmac_interrupt()
1679 		dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1687 		for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_interrupt()
1691 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1695 			dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1709 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1755 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()  local
1764 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1776 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()  local
1787 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1798 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()  local
1804 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1805 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1857 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()  local
1863 		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);  in at_xdmac_free_chan_resources()
1872 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_prepare()  local
1875 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1891 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_suspend()  local
1894 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1906 	atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in atmel_xdmac_suspend()
1908 	at_xdmac_off(atxdmac);  in atmel_xdmac_suspend()
1909 	clk_disable_unprepare(atxdmac->clk);  in atmel_xdmac_suspend()
1915 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_resume()  local
1921 	ret = clk_prepare_enable(atxdmac->clk);  in atmel_xdmac_resume()
1926 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in atmel_xdmac_resume()
1927 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1932 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);  in atmel_xdmac_resume()
1933 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
1943 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
1952 	struct at_xdmac	*atxdmac;  in at_xdmac_probe()  local
1978 	size = sizeof(*atxdmac);  in at_xdmac_probe()
1980 	atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in at_xdmac_probe()
1981 	if (!atxdmac) {  in at_xdmac_probe()
1986 	atxdmac->regs = base;  in at_xdmac_probe()
1987 	atxdmac->irq = irq;  in at_xdmac_probe()
1989 	atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");  in at_xdmac_probe()
1990 	if (IS_ERR(atxdmac->clk)) {  in at_xdmac_probe()
1992 		return PTR_ERR(atxdmac->clk);  in at_xdmac_probe()
1996 	ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);  in at_xdmac_probe()
2002 	ret = clk_prepare_enable(atxdmac->clk);  in at_xdmac_probe()
2008 	atxdmac->at_xdmac_desc_pool =  in at_xdmac_probe()
2011 	if (!atxdmac->at_xdmac_desc_pool) {  in at_xdmac_probe()
2017 	dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2018 	dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2019 	dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2020 	dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2021 	dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2022 	dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2027 	dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
2028 	atxdmac->dma.dev				= &pdev->dev;  in at_xdmac_probe()
2029 	atxdmac->dma.device_alloc_chan_resources	= at_xdmac_alloc_chan_resources;  in at_xdmac_probe()
2030 	atxdmac->dma.device_free_chan_resources		= at_xdmac_free_chan_resources;  in at_xdmac_probe()
2031 	atxdmac->dma.device_tx_status			= at_xdmac_tx_status;  in at_xdmac_probe()
2032 	atxdmac->dma.device_issue_pending		= at_xdmac_issue_pending;  in at_xdmac_probe()
2033 	atxdmac->dma.device_prep_dma_cyclic		= at_xdmac_prep_dma_cyclic;  in at_xdmac_probe()
2034 	atxdmac->dma.device_prep_interleaved_dma	= at_xdmac_prep_interleaved;  in at_xdmac_probe()
2035 	atxdmac->dma.device_prep_dma_memcpy		= at_xdmac_prep_dma_memcpy;  in at_xdmac_probe()
2036 	atxdmac->dma.device_prep_dma_memset		= at_xdmac_prep_dma_memset;  in at_xdmac_probe()
2037 	atxdmac->dma.device_prep_dma_memset_sg		= at_xdmac_prep_dma_memset_sg;  in at_xdmac_probe()
2038 	atxdmac->dma.device_prep_slave_sg		= at_xdmac_prep_slave_sg;  in at_xdmac_probe()
2039 	atxdmac->dma.device_config			= at_xdmac_device_config;  in at_xdmac_probe()
2040 	atxdmac->dma.device_pause			= at_xdmac_device_pause;  in at_xdmac_probe()
2041 	atxdmac->dma.device_resume			= at_xdmac_device_resume;  in at_xdmac_probe()
2042 	atxdmac->dma.device_terminate_all		= at_xdmac_device_terminate_all;  in at_xdmac_probe()
2043 	atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2044 	atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2045 	atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_xdmac_probe()
2046 	atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_xdmac_probe()
2049 	at_xdmac_off(atxdmac);  in at_xdmac_probe()
2052 	INIT_LIST_HEAD(&atxdmac->dma.channels);  in at_xdmac_probe()
2054 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
2056 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2058 			      &atxdmac->dma.channels);  in at_xdmac_probe()
2060 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2072 	platform_set_drvdata(pdev, atxdmac);  in at_xdmac_probe()
2074 	ret = dma_async_device_register(&atxdmac->dma);  in at_xdmac_probe()
2081 					 at_xdmac_xlate, atxdmac);  in at_xdmac_probe()
2088 		 nr_channels, atxdmac->regs);  in at_xdmac_probe()
2093 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_probe()
2095 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_probe()
2097 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_probe()
2103 	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);  in at_xdmac_remove()  local
2106 	at_xdmac_off(atxdmac);  in at_xdmac_remove()
2108 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_remove()
2109 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_remove()
2111 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_remove()
2113 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_remove()
2114 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()