189 	struct tegra_dma	*tdma;  member
232 static inline void tdma_write(struct tegra_dma *tdma, u32 reg, u32 val)  in tdma_write()  argument
234 	writel(val, tdma->base_addr + reg);  in tdma_write()
237 static inline u32 tdma_read(struct tegra_dma *tdma, u32 reg)  in tdma_read()  argument
239 	return readl(tdma->base_addr + reg);  in tdma_read()
362 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_global_pause()  local
364 	spin_lock(&tdma->global_lock);  in tegra_dma_global_pause()
366 	if (tdc->tdma->global_pause_count == 0) {  in tegra_dma_global_pause()
367 		tdma_write(tdma, TEGRA_APBDMA_GENERAL, 0);  in tegra_dma_global_pause()
372 	tdc->tdma->global_pause_count++;  in tegra_dma_global_pause()
374 	spin_unlock(&tdma->global_lock);  in tegra_dma_global_pause()
379 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_global_resume()  local
381 	spin_lock(&tdma->global_lock);  in tegra_dma_global_resume()
383 	if (WARN_ON(tdc->tdma->global_pause_count == 0))  in tegra_dma_global_resume()
386 	if (--tdc->tdma->global_pause_count == 0)  in tegra_dma_global_resume()
387 		tdma_write(tdma, TEGRA_APBDMA_GENERAL,  in tegra_dma_global_resume()
391 	spin_unlock(&tdma->global_lock);  in tegra_dma_global_resume()
397 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_pause()  local
399 	if (tdma->chip_data->support_channel_pause) {  in tegra_dma_pause()
411 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_resume()  local
413 	if (tdma->chip_data->support_channel_pause) {  in tegra_dma_resume()
453 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_start()
494 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_configure_for_next()
776 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_terminate_all()
813 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_sg_bytes_xferred()
818 	if (!tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_sg_bytes_xferred()
998 	if (tdc->tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_prep_wcount()
1075 				(len > tdc->tdma->chip_data->max_dma_count)) {  in tegra_dma_prep_slave_sg()
1174 			(len > tdc->tdma->chip_data->max_dma_count)) {  in tegra_dma_prep_dma_cyclic()
1266 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_alloc_chan_resources()  local
1272 	ret = pm_runtime_get_sync(tdma->dev);  in tegra_dma_alloc_chan_resources()
1282 	struct tegra_dma *tdma = tdc->tdma;  in tegra_dma_free_chan_resources()  local
1318 	pm_runtime_put(tdma->dev);  in tegra_dma_free_chan_resources()
1326 	struct tegra_dma *tdma = ofdma->of_dma_data;  in tegra_dma_of_xlate()  local
1331 		dev_err(tdma->dev, "Invalid slave id: %d\n", dma_spec->args[0]);  in tegra_dma_of_xlate()
1335 	chan = dma_get_any_slave_channel(&tdma->dma_dev);  in tegra_dma_of_xlate()
1384 	struct tegra_dma *tdma;  in tegra_dma_probe()  local
1395 	tdma = devm_kzalloc(&pdev->dev,  in tegra_dma_probe()
1396 			    struct_size(tdma, channels, cdata->nr_channels),  in tegra_dma_probe()
1398 	if (!tdma)  in tegra_dma_probe()
1401 	tdma->dev = &pdev->dev;  in tegra_dma_probe()
1402 	tdma->chip_data = cdata;  in tegra_dma_probe()
1403 	platform_set_drvdata(pdev, tdma);  in tegra_dma_probe()
1406 	tdma->base_addr = devm_ioremap_resource(&pdev->dev, res);  in tegra_dma_probe()
1407 	if (IS_ERR(tdma->base_addr))  in tegra_dma_probe()
1408 		return PTR_ERR(tdma->base_addr);  in tegra_dma_probe()
1410 	tdma->dma_clk = devm_clk_get(&pdev->dev, NULL);  in tegra_dma_probe()
1411 	if (IS_ERR(tdma->dma_clk)) {  in tegra_dma_probe()
1413 		return PTR_ERR(tdma->dma_clk);  in tegra_dma_probe()
1416 	tdma->rst = devm_reset_control_get(&pdev->dev, "dma");  in tegra_dma_probe()
1417 	if (IS_ERR(tdma->rst)) {  in tegra_dma_probe()
1419 		return PTR_ERR(tdma->rst);  in tegra_dma_probe()
1422 	spin_lock_init(&tdma->global_lock);  in tegra_dma_probe()
1436 	reset_control_assert(tdma->rst);  in tegra_dma_probe()
1438 	reset_control_deassert(tdma->rst);  in tegra_dma_probe()
1441 	tdma_write(tdma, TEGRA_APBDMA_GENERAL, TEGRA_APBDMA_GENERAL_ENABLE);  in tegra_dma_probe()
1442 	tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);  in tegra_dma_probe()
1443 	tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);  in tegra_dma_probe()
1447 	INIT_LIST_HEAD(&tdma->dma_dev.channels);  in tegra_dma_probe()
1449 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_probe()
1451 		tdc->chan_addr = tdma->base_addr +  in tegra_dma_probe()
1471 		tdc->dma_chan.device = &tdma->dma_dev;  in tegra_dma_probe()
1474 				&tdma->dma_dev.channels);  in tegra_dma_probe()
1475 		tdc->tdma = tdma;  in tegra_dma_probe()
1489 	dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);  in tegra_dma_probe()
1490 	dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);  in tegra_dma_probe()
1491 	dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);  in tegra_dma_probe()
1493 	tdma->global_pause_count = 0;  in tegra_dma_probe()
1494 	tdma->dma_dev.dev = &pdev->dev;  in tegra_dma_probe()
1495 	tdma->dma_dev.device_alloc_chan_resources =  in tegra_dma_probe()
1497 	tdma->dma_dev.device_free_chan_resources =  in tegra_dma_probe()
1499 	tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;  in tegra_dma_probe()
1500 	tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;  in tegra_dma_probe()
1501 	tdma->dma_dev.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |  in tegra_dma_probe()
1505 	tdma->dma_dev.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |  in tegra_dma_probe()
1509 	tdma->dma_dev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);  in tegra_dma_probe()
1510 	tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;  in tegra_dma_probe()
1511 	tdma->dma_dev.device_config = tegra_dma_slave_config;  in tegra_dma_probe()
1512 	tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;  in tegra_dma_probe()
1513 	tdma->dma_dev.device_tx_status = tegra_dma_tx_status;  in tegra_dma_probe()
1514 	tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;  in tegra_dma_probe()
1516 	ret = dma_async_device_register(&tdma->dma_dev);  in tegra_dma_probe()
1524 					 tegra_dma_of_xlate, tdma);  in tegra_dma_probe()
1536 	dma_async_device_unregister(&tdma->dma_dev);  in tegra_dma_probe()
1539 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_probe()
1553 	struct tegra_dma *tdma = platform_get_drvdata(pdev);  in tegra_dma_remove()  local
1557 	dma_async_device_unregister(&tdma->dma_dev);  in tegra_dma_remove()
1559 	for (i = 0; i < tdma->chip_data->nr_channels; ++i) {  in tegra_dma_remove()
1560 		tdc = &tdma->channels[i];  in tegra_dma_remove()
1574 	struct tegra_dma *tdma = dev_get_drvdata(dev);  in tegra_dma_runtime_suspend()  local
1577 	tdma->reg_gen = tdma_read(tdma, TEGRA_APBDMA_GENERAL);  in tegra_dma_runtime_suspend()
1578 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {  in tegra_dma_runtime_suspend()
1579 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_runtime_suspend()
1591 		if (tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_runtime_suspend()
1596 	clk_disable_unprepare(tdma->dma_clk);  in tegra_dma_runtime_suspend()
1603 	struct tegra_dma *tdma = dev_get_drvdata(dev);  in tegra_dma_runtime_resume()  local
1606 	ret = clk_prepare_enable(tdma->dma_clk);  in tegra_dma_runtime_resume()
1612 	tdma_write(tdma, TEGRA_APBDMA_GENERAL, tdma->reg_gen);  in tegra_dma_runtime_resume()
1613 	tdma_write(tdma, TEGRA_APBDMA_CONTROL, 0);  in tegra_dma_runtime_resume()
1614 	tdma_write(tdma, TEGRA_APBDMA_IRQ_MASK_SET, 0xFFFFFFFFul);  in tegra_dma_runtime_resume()
1616 	for (i = 0; i < tdma->chip_data->nr_channels; i++) {  in tegra_dma_runtime_resume()
1617 		struct tegra_dma_channel *tdc = &tdma->channels[i];  in tegra_dma_runtime_resume()
1624 		if (tdma->chip_data->support_separate_wcount_reg)  in tegra_dma_runtime_resume()