257 static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)  in at_xdmac_chan_reg_base()  argument
259 	return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);  in at_xdmac_chan_reg_base()
262 #define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))  argument
263 #define at_xdmac_write(atxdmac, reg, value) \  argument
264 	writel_relaxed((value), (atxdmac)->regs + (reg))
326 static void at_xdmac_off(struct at_xdmac *atxdmac)  in at_xdmac_off()  argument
328 	at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);  in at_xdmac_off()
331 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS))  in at_xdmac_off()
334 	at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);  in at_xdmac_off()
341 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()  local
402 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
406 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
443 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_alloc_desc()  local
446 	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);  in at_xdmac_alloc_desc()
513 	struct at_xdmac		*atxdmac = of_dma->of_dma_data;  in at_xdmac_xlate()  local
516 	struct device		*dev = atxdmac->dma.dev;  in at_xdmac_xlate()
523 	chan = dma_get_any_slave_channel(&atxdmac->dma);  in at_xdmac_xlate()
1393 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()  local
1440 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1498 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1629 	struct at_xdmac		*atxdmac = (struct at_xdmac *)dev_id;  in at_xdmac_interrupt()  local
1636 		imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in at_xdmac_interrupt()
1637 		status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);  in at_xdmac_interrupt()
1640 		dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1648 		for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_interrupt()
1652 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1656 			dev_vdbg(atxdmac->dma.dev,  in at_xdmac_interrupt()
1670 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1712 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()  local
1721 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1733 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()  local
1744 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1755 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()  local
1761 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1762 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1821 	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);  in at_xdmac_free_chan_resources()  local
1827 		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);  in at_xdmac_free_chan_resources()
1836 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_prepare()  local
1839 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_prepare()
1855 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_suspend()  local
1858 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_suspend()
1870 	atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);  in atmel_xdmac_suspend()
1872 	at_xdmac_off(atxdmac);  in atmel_xdmac_suspend()
1873 	clk_disable_unprepare(atxdmac->clk);  in atmel_xdmac_suspend()
1879 	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);  in atmel_xdmac_resume()  local
1885 	ret = clk_prepare_enable(atxdmac->clk);  in atmel_xdmac_resume()
1890 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in atmel_xdmac_resume()
1891 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1896 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);  in atmel_xdmac_resume()
1897 	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {  in atmel_xdmac_resume()
1907 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
1917 	struct at_xdmac	*atxdmac;  in at_xdmac_probe()  local
1947 	size = sizeof(*atxdmac);  in at_xdmac_probe()
1949 	atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);  in at_xdmac_probe()
1950 	if (!atxdmac) {  in at_xdmac_probe()
1955 	atxdmac->regs = base;  in at_xdmac_probe()
1956 	atxdmac->irq = irq;  in at_xdmac_probe()
1958 	atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");  in at_xdmac_probe()
1959 	if (IS_ERR(atxdmac->clk)) {  in at_xdmac_probe()
1961 		return PTR_ERR(atxdmac->clk);  in at_xdmac_probe()
1965 	ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);  in at_xdmac_probe()
1971 	ret = clk_prepare_enable(atxdmac->clk);  in at_xdmac_probe()
1977 	atxdmac->at_xdmac_desc_pool =  in at_xdmac_probe()
1980 	if (!atxdmac->at_xdmac_desc_pool) {  in at_xdmac_probe()
1986 	dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1987 	dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1988 	dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1989 	dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1990 	dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1991 	dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1996 	dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);  in at_xdmac_probe()
1997 	atxdmac->dma.dev				= &pdev->dev;  in at_xdmac_probe()
1998 	atxdmac->dma.device_alloc_chan_resources	= at_xdmac_alloc_chan_resources;  in at_xdmac_probe()
1999 	atxdmac->dma.device_free_chan_resources		= at_xdmac_free_chan_resources;  in at_xdmac_probe()
2000 	atxdmac->dma.device_tx_status			= at_xdmac_tx_status;  in at_xdmac_probe()
2001 	atxdmac->dma.device_issue_pending		= at_xdmac_issue_pending;  in at_xdmac_probe()
2002 	atxdmac->dma.device_prep_dma_cyclic		= at_xdmac_prep_dma_cyclic;  in at_xdmac_probe()
2003 	atxdmac->dma.device_prep_interleaved_dma	= at_xdmac_prep_interleaved;  in at_xdmac_probe()
2004 	atxdmac->dma.device_prep_dma_memcpy		= at_xdmac_prep_dma_memcpy;  in at_xdmac_probe()
2005 	atxdmac->dma.device_prep_dma_memset		= at_xdmac_prep_dma_memset;  in at_xdmac_probe()
2006 	atxdmac->dma.device_prep_dma_memset_sg		= at_xdmac_prep_dma_memset_sg;  in at_xdmac_probe()
2007 	atxdmac->dma.device_prep_slave_sg		= at_xdmac_prep_slave_sg;  in at_xdmac_probe()
2008 	atxdmac->dma.device_config			= at_xdmac_device_config;  in at_xdmac_probe()
2009 	atxdmac->dma.device_pause			= at_xdmac_device_pause;  in at_xdmac_probe()
2010 	atxdmac->dma.device_resume			= at_xdmac_device_resume;  in at_xdmac_probe()
2011 	atxdmac->dma.device_terminate_all		= at_xdmac_device_terminate_all;  in at_xdmac_probe()
2012 	atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2013 	atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;  in at_xdmac_probe()
2014 	atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_xdmac_probe()
2015 	atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_xdmac_probe()
2018 	at_xdmac_off(atxdmac);  in at_xdmac_probe()
2021 	INIT_LIST_HEAD(&atxdmac->dma.channels);  in at_xdmac_probe()
2023 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()
2025 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2027 			      &atxdmac->dma.channels);  in at_xdmac_probe()
2029 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2042 	platform_set_drvdata(pdev, atxdmac);  in at_xdmac_probe()
2044 	ret = dma_async_device_register(&atxdmac->dma);  in at_xdmac_probe()
2051 					 at_xdmac_xlate, atxdmac);  in at_xdmac_probe()
2058 		 nr_channels, atxdmac->regs);  in at_xdmac_probe()
2063 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_probe()
2065 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_probe()
2067 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_probe()
2073 	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);  in at_xdmac_remove()  local
2076 	at_xdmac_off(atxdmac);  in at_xdmac_remove()
2078 	dma_async_device_unregister(&atxdmac->dma);  in at_xdmac_remove()
2079 	clk_disable_unprepare(atxdmac->clk);  in at_xdmac_remove()
2081 	free_irq(atxdmac->irq, atxdmac);  in at_xdmac_remove()
2083 	for (i = 0; i < atxdmac->dma.chancnt; i++) {  in at_xdmac_remove()
2084 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()