111 	struct at_dma	*atdma = to_at_dma(chan->device);  in atc_alloc_descriptor()  local
114 	desc = dma_pool_zalloc(atdma->dma_desc_pool, gfp_flags, &phys);  in atc_alloc_descriptor()
226 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_dostart()  local
255 	dma_writel(atdma, CHER, atchan->mask);  in atc_dostart()
450 	struct at_dma			*atdma = to_at_dma(atchan->chan_common.device);  in atc_chain_complete()  local
461 		dma_pool_free(atdma->memset_pool, desc->memset_vaddr,  in atc_chain_complete()
624 	struct at_dma		*atdma = (struct at_dma *)dev_id;  in at_dma_interrupt()  local
631 		imr = dma_readl(atdma, EBCIMR);  in at_dma_interrupt()
632 		status = dma_readl(atdma, EBCISR);  in at_dma_interrupt()
638 		dev_vdbg(atdma->dma_common.dev,  in at_dma_interrupt()
642 		for (i = 0; i < atdma->dma_common.chancnt; i++) {  in at_dma_interrupt()
643 			atchan = &atdma->chan[i];  in at_dma_interrupt()
647 					dma_writel(atdma, CHDR,  in at_dma_interrupt()
930 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset()  local
949 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);  in atc_prep_dma_memset()
979 	dma_pool_free(atdma->memset_pool, vaddr, paddr);  in atc_prep_dma_memset()
990 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_prep_dma_memset_sg()  local
1007 	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);  in atc_prep_dma_memset_sg()
1386 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_pause()  local
1396 	dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));  in atc_pause()
1407 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_resume()  local
1420 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));  in atc_resume()
1431 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_terminate_all()  local
1449 	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);  in atc_terminate_all()
1452 	while (dma_readl(atdma, CHSR) & atchan->mask)  in atc_terminate_all()
1552 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_alloc_chan_resources()  local
1576 		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);  in atc_alloc_chan_resources()
1592 			dev_err(atdma->dma_common.dev,  in atc_alloc_chan_resources()
1622 	struct at_dma		*atdma = to_at_dma(chan->device);  in atc_free_chan_resources()  local
1638 		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);  in atc_free_chan_resources()
1782 static void at_dma_off(struct at_dma *atdma)  in at_dma_off()  argument
1784 	dma_writel(atdma, EN, 0);  in at_dma_off()
1787 	dma_writel(atdma, EBCIDR, -1L);  in at_dma_off()
1790 	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)  in at_dma_off()
1797 	struct at_dma		*atdma;  in at_dma_probe()  local
1828 	atdma = kzalloc(size, GFP_KERNEL);  in at_dma_probe()
1829 	if (!atdma)  in at_dma_probe()
1833 	atdma->dma_common.cap_mask = plat_dat->cap_mask;  in at_dma_probe()
1834 	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;  in at_dma_probe()
1842 	atdma->regs = ioremap(io->start, size);  in at_dma_probe()
1843 	if (!atdma->regs) {  in at_dma_probe()
1848 	atdma->clk = clk_get(&pdev->dev, "dma_clk");  in at_dma_probe()
1849 	if (IS_ERR(atdma->clk)) {  in at_dma_probe()
1850 		err = PTR_ERR(atdma->clk);  in at_dma_probe()
1853 	err = clk_prepare_enable(atdma->clk);  in at_dma_probe()
1858 	at_dma_off(atdma);  in at_dma_probe()
1860 	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);  in at_dma_probe()
1864 	platform_set_drvdata(pdev, atdma);  in at_dma_probe()
1867 	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",  in at_dma_probe()
1870 	if (!atdma->dma_desc_pool) {  in at_dma_probe()
1877 	atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",  in at_dma_probe()
1879 	if (!atdma->memset_pool) {  in at_dma_probe()
1886 	while (dma_readl(atdma, EBCISR))  in at_dma_probe()
1890 	INIT_LIST_HEAD(&atdma->dma_common.channels);  in at_dma_probe()
1892 		struct at_dma_chan	*atchan = &atdma->chan[i];  in at_dma_probe()
1896 		atchan->chan_common.device = &atdma->dma_common;  in at_dma_probe()
1899 				&atdma->dma_common.channels);  in at_dma_probe()
1901 		atchan->ch_regs = atdma->regs + ch_regs(i);  in at_dma_probe()
1911 		atc_enable_chan_irq(atdma, i);  in at_dma_probe()
1915 	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;  in at_dma_probe()
1916 	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;  in at_dma_probe()
1917 	atdma->dma_common.device_tx_status = atc_tx_status;  in at_dma_probe()
1918 	atdma->dma_common.device_issue_pending = atc_issue_pending;  in at_dma_probe()
1919 	atdma->dma_common.dev = &pdev->dev;  in at_dma_probe()
1922 	if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask))  in at_dma_probe()
1923 		atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved;  in at_dma_probe()
1925 	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))  in at_dma_probe()
1926 		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;  in at_dma_probe()
1928 	if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {  in at_dma_probe()
1929 		atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;  in at_dma_probe()
1930 		atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg;  in at_dma_probe()
1931 		atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;  in at_dma_probe()
1934 	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {  in at_dma_probe()
1935 		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;  in at_dma_probe()
1937 		dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);  in at_dma_probe()
1938 		atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;  in at_dma_probe()
1939 		atdma->dma_common.device_config = atc_config;  in at_dma_probe()
1940 		atdma->dma_common.device_pause = atc_pause;  in at_dma_probe()
1941 		atdma->dma_common.device_resume = atc_resume;  in at_dma_probe()
1942 		atdma->dma_common.device_terminate_all = atc_terminate_all;  in at_dma_probe()
1943 		atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1944 		atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;  in at_dma_probe()
1945 		atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in at_dma_probe()
1946 		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in at_dma_probe()
1949 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_probe()
1952 	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",  in at_dma_probe()
1953 	  dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",  in at_dma_probe()
1954 	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",  in at_dma_probe()
1957 	dma_async_device_register(&atdma->dma_common);  in at_dma_probe()
1966 						 at_dma_xlate, atdma);  in at_dma_probe()
1976 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_probe()
1977 	dma_pool_destroy(atdma->memset_pool);  in at_dma_probe()
1979 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_probe()
1981 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_probe()
1983 	clk_disable_unprepare(atdma->clk);  in at_dma_probe()
1985 	clk_put(atdma->clk);  in at_dma_probe()
1987 	iounmap(atdma->regs);  in at_dma_probe()
1988 	atdma->regs = NULL;  in at_dma_probe()
1992 	kfree(atdma);  in at_dma_probe()
1998 	struct at_dma		*atdma = platform_get_drvdata(pdev);  in at_dma_remove()  local
2002 	at_dma_off(atdma);  in at_dma_remove()
2003 	dma_async_device_unregister(&atdma->dma_common);  in at_dma_remove()
2005 	dma_pool_destroy(atdma->memset_pool);  in at_dma_remove()
2006 	dma_pool_destroy(atdma->dma_desc_pool);  in at_dma_remove()
2007 	free_irq(platform_get_irq(pdev, 0), atdma);  in at_dma_remove()
2009 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_remove()
2014 		atc_disable_chan_irq(atdma, chan->chan_id);  in at_dma_remove()
2020 	clk_disable_unprepare(atdma->clk);  in at_dma_remove()
2021 	clk_put(atdma->clk);  in at_dma_remove()
2023 	iounmap(atdma->regs);  in at_dma_remove()
2024 	atdma->regs = NULL;  in at_dma_remove()
2029 	kfree(atdma);  in at_dma_remove()
2036 	struct at_dma	*atdma = platform_get_drvdata(pdev);  in at_dma_shutdown()  local
2039 	clk_disable_unprepare(atdma->clk);  in at_dma_shutdown()
2044 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_prepare()  local
2047 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_prepare()
2078 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_suspend_noirq()  local
2082 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_suspend_noirq()
2090 	atdma->save_imr = dma_readl(atdma, EBCIMR);  in at_dma_suspend_noirq()
2093 	at_dma_off(atdma);  in at_dma_suspend_noirq()
2094 	clk_disable_unprepare(atdma->clk);  in at_dma_suspend_noirq()
2100 	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);  in atc_resume_cyclic()  local
2109 	dma_writel(atdma, CHER, atchan->mask);  in atc_resume_cyclic()
2119 	struct at_dma *atdma = dev_get_drvdata(dev);  in at_dma_resume_noirq()  local
2123 	clk_prepare_enable(atdma->clk);  in at_dma_resume_noirq()
2124 	dma_writel(atdma, EN, AT_DMA_ENABLE);  in at_dma_resume_noirq()
2127 	while (dma_readl(atdma, EBCISR))  in at_dma_resume_noirq()
2131 	dma_writel(atdma, EBCIER, atdma->save_imr);  in at_dma_resume_noirq()
2132 	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,  in at_dma_resume_noirq()