1537 	struct dma_pl330_chan *pch;  in dma_pl330_rqcb()  local
1543 	pch = desc->pchan;  in dma_pl330_rqcb()
1546 	if (!pch)  in dma_pl330_rqcb()
1549 	spin_lock_irqsave(&pch->lock, flags);  in dma_pl330_rqcb()
1553 	spin_unlock_irqrestore(&pch->lock, flags);  in dma_pl330_rqcb()
1555 	tasklet_schedule(&pch->task);  in dma_pl330_rqcb()
2019 static inline void fill_queue(struct dma_pl330_chan *pch)  in fill_queue()  argument
2024 	list_for_each_entry(desc, &pch->work_list, node) {  in fill_queue()
2030 		ret = pl330_submit_req(pch->thread, desc);  in fill_queue()
2039 			dev_err(pch->dmac->ddma.dev, "%s:%d Bad Desc(%d)\n",  in fill_queue()
2041 			tasklet_schedule(&pch->task);  in fill_queue()
2048 	struct dma_pl330_chan *pch = (struct dma_pl330_chan *)data;  in pl330_tasklet()  local
2053 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
2056 	list_for_each_entry_safe(desc, _dt, &pch->work_list, node)  in pl330_tasklet()
2058 			if (!pch->cyclic)  in pl330_tasklet()
2060 			list_move_tail(&desc->node, &pch->completed_list);  in pl330_tasklet()
2064 	fill_queue(pch);  in pl330_tasklet()
2066 	if (list_empty(&pch->work_list)) {  in pl330_tasklet()
2067 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2068 		_stop(pch->thread);  in pl330_tasklet()
2069 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2071 		pch->active = false;  in pl330_tasklet()
2074 		spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2075 		_start(pch->thread);  in pl330_tasklet()
2076 		spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2079 	while (!list_empty(&pch->completed_list)) {  in pl330_tasklet()
2082 		desc = list_first_entry(&pch->completed_list,  in pl330_tasklet()
2087 		if (pch->cyclic) {  in pl330_tasklet()
2089 			list_move_tail(&desc->node, &pch->work_list);  in pl330_tasklet()
2091 				pch->active = true;  in pl330_tasklet()
2092 				spin_lock(&pch->thread->dmac->lock);  in pl330_tasklet()
2093 				_start(pch->thread);  in pl330_tasklet()
2094 				spin_unlock(&pch->thread->dmac->lock);  in pl330_tasklet()
2099 			list_move_tail(&desc->node, &pch->dmac->desc_pool);  in pl330_tasklet()
2105 			spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2107 			spin_lock_irqsave(&pch->lock, flags);  in pl330_tasklet()
2110 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tasklet()
2114 		pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_tasklet()
2115 		pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_tasklet()
2141 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_alloc_chan_resources()  local
2142 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_alloc_chan_resources()
2148 	pch->cyclic = false;  in pl330_alloc_chan_resources()
2150 	pch->thread = pl330_request_channel(pl330);  in pl330_alloc_chan_resources()
2151 	if (!pch->thread) {  in pl330_alloc_chan_resources()
2156 	tasklet_init(&pch->task, pl330_tasklet, (unsigned long) pch);  in pl330_alloc_chan_resources()
2182 static void pl330_unprep_slave_fifo(struct dma_pl330_chan *pch)  in pl330_unprep_slave_fifo()  argument
2184 	if (pch->dir != DMA_NONE)  in pl330_unprep_slave_fifo()
2185 		dma_unmap_resource(pch->chan.device->dev, pch->fifo_dma,  in pl330_unprep_slave_fifo()
2186 				   1 << pch->burst_sz, pch->dir, 0);  in pl330_unprep_slave_fifo()
2187 	pch->dir = DMA_NONE;  in pl330_unprep_slave_fifo()
2191 static bool pl330_prep_slave_fifo(struct dma_pl330_chan *pch,  in pl330_prep_slave_fifo()  argument
2194 	struct device *dev = pch->chan.device->dev;  in pl330_prep_slave_fifo()
2198 	if (pch->dir == dma_dir)  in pl330_prep_slave_fifo()
2201 	pl330_unprep_slave_fifo(pch);  in pl330_prep_slave_fifo()
2202 	pch->fifo_dma = dma_map_resource(dev, pch->fifo_addr,  in pl330_prep_slave_fifo()
2203 					 1 << pch->burst_sz, dma_dir, 0);  in pl330_prep_slave_fifo()
2204 	if (dma_mapping_error(dev, pch->fifo_dma))  in pl330_prep_slave_fifo()
2207 	pch->dir = dma_dir;  in pl330_prep_slave_fifo()
2226 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_config()  local
2228 	pl330_unprep_slave_fifo(pch);  in pl330_config()
2231 			pch->fifo_addr = slave_config->dst_addr;  in pl330_config()
2233 			pch->burst_sz = __ffs(slave_config->dst_addr_width);  in pl330_config()
2234 		pch->burst_len = fixup_burst_len(slave_config->dst_maxburst,  in pl330_config()
2235 			pch->dmac->quirks);  in pl330_config()
2238 			pch->fifo_addr = slave_config->src_addr;  in pl330_config()
2240 			pch->burst_sz = __ffs(slave_config->src_addr_width);  in pl330_config()
2241 		pch->burst_len = fixup_burst_len(slave_config->src_maxburst,  in pl330_config()
2242 			pch->dmac->quirks);  in pl330_config()
2250 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_terminate_all()  local
2253 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_terminate_all()
2258 	spin_lock_irqsave(&pch->lock, flags);  in pl330_terminate_all()
2261 	_stop(pch->thread);  in pl330_terminate_all()
2262 	pch->thread->req[0].desc = NULL;  in pl330_terminate_all()
2263 	pch->thread->req[1].desc = NULL;  in pl330_terminate_all()
2264 	pch->thread->req_running = -1;  in pl330_terminate_all()
2267 	power_down = pch->active;  in pl330_terminate_all()
2268 	pch->active = false;  in pl330_terminate_all()
2271 	list_for_each_entry(desc, &pch->submitted_list, node) {  in pl330_terminate_all()
2276 	list_for_each_entry(desc, &pch->work_list , node) {  in pl330_terminate_all()
2281 	list_splice_tail_init(&pch->submitted_list, &pl330->desc_pool);  in pl330_terminate_all()
2282 	list_splice_tail_init(&pch->work_list, &pl330->desc_pool);  in pl330_terminate_all()
2283 	list_splice_tail_init(&pch->completed_list, &pl330->desc_pool);  in pl330_terminate_all()
2284 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_terminate_all()
2302 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_pause()  local
2303 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_pause()
2307 	spin_lock_irqsave(&pch->lock, flags);  in pl330_pause()
2310 	_stop(pch->thread);  in pl330_pause()
2313 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_pause()
2322 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_free_chan_resources()  local
2323 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_free_chan_resources()
2326 	tasklet_kill(&pch->task);  in pl330_free_chan_resources()
2328 	pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2331 	pl330_release_channel(pch->thread);  in pl330_free_chan_resources()
2332 	pch->thread = NULL;  in pl330_free_chan_resources()
2334 	if (pch->cyclic)  in pl330_free_chan_resources()
2335 		list_splice_tail_init(&pch->work_list, &pch->dmac->desc_pool);  in pl330_free_chan_resources()
2338 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2339 	pm_runtime_put_autosuspend(pch->dmac->ddma.dev);  in pl330_free_chan_resources()
2340 	pl330_unprep_slave_fifo(pch);  in pl330_free_chan_resources()
2343 static int pl330_get_current_xferred_count(struct dma_pl330_chan *pch,  in pl330_get_current_xferred_count()  argument
2346 	struct pl330_thread *thrd = pch->thread;  in pl330_get_current_xferred_count()
2347 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_current_xferred_count()
2360 	pm_runtime_mark_last_busy(pch->dmac->ddma.dev);  in pl330_get_current_xferred_count()
2377 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_tx_status()  local
2388 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_status()
2389 	spin_lock(&pch->thread->dmac->lock);  in pl330_tx_status()
2391 	if (pch->thread->req_running != -1)  in pl330_tx_status()
2392 		running = pch->thread->req[pch->thread->req_running].desc;  in pl330_tx_status()
2394 	last_enq = pch->thread->req[pch->thread->lstenq].desc;  in pl330_tx_status()
2397 	list_for_each_entry(desc, &pch->work_list, node) {  in pl330_tx_status()
2402 				pl330_get_current_xferred_count(pch, desc);  in pl330_tx_status()
2432 	spin_unlock(&pch->thread->dmac->lock);  in pl330_tx_status()
2433 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_status()
2443 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_issue_pending()  local
2446 	spin_lock_irqsave(&pch->lock, flags);  in pl330_issue_pending()
2447 	if (list_empty(&pch->work_list)) {  in pl330_issue_pending()
2453 		WARN_ON(list_empty(&pch->submitted_list));  in pl330_issue_pending()
2454 		pch->active = true;  in pl330_issue_pending()
2455 		pm_runtime_get_sync(pch->dmac->ddma.dev);  in pl330_issue_pending()
2457 	list_splice_tail_init(&pch->submitted_list, &pch->work_list);  in pl330_issue_pending()
2458 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_issue_pending()
2460 	pl330_tasklet((unsigned long)pch);  in pl330_issue_pending()
2471 	struct dma_pl330_chan *pch = to_pchan(tx->chan);  in pl330_tx_submit()  local
2475 	spin_lock_irqsave(&pch->lock, flags);  in pl330_tx_submit()
2480 		if (pch->cyclic) {  in pl330_tx_submit()
2488 		list_move_tail(&desc->node, &pch->submitted_list);  in pl330_tx_submit()
2493 	list_add_tail(&last->node, &pch->submitted_list);  in pl330_tx_submit()
2494 	spin_unlock_irqrestore(&pch->lock, flags);  in pl330_tx_submit()
2556 static struct dma_pl330_desc *pl330_get_desc(struct dma_pl330_chan *pch)  in pl330_get_desc()  argument
2558 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_get_desc()
2559 	u8 *peri_id = pch->chan.private;  in pl330_get_desc()
2578 	desc->pchan = pch;  in pl330_get_desc()
2582 	desc->peri = peri_id ? pch->chan.chan_id : 0;  in pl330_get_desc()
2583 	desc->rqcfg.pcfg = &pch->dmac->pcfg;  in pl330_get_desc()
2585 	dma_async_tx_descriptor_init(&desc->txd, &pch->chan);  in pl330_get_desc()
2599 __pl330_prep_dma_memcpy(struct dma_pl330_chan *pch, dma_addr_t dst,  in __pl330_prep_dma_memcpy()  argument
2602 	struct dma_pl330_desc *desc = pl330_get_desc(pch);  in __pl330_prep_dma_memcpy()
2605 		dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in __pl330_prep_dma_memcpy()
2628 	struct dma_pl330_chan *pch = desc->pchan;  in get_burst_len()  local
2629 	struct pl330_dmac *pl330 = pch->dmac;  in get_burst_len()
2649 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_cyclic()  local
2650 	struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_dma_cyclic()
2659 		dev_err(pch->dmac->ddma.dev, "%s:%d Invalid dma direction\n",  in pl330_prep_dma_cyclic()
2664 	if (!pl330_prep_slave_fifo(pch, direction))  in pl330_prep_dma_cyclic()
2668 		desc = pl330_get_desc(pch);  in pl330_prep_dma_cyclic()
2670 			dev_err(pch->dmac->ddma.dev, "%s:%d Unable to fetch desc\n",  in pl330_prep_dma_cyclic()
2696 			dst = pch->fifo_dma;  in pl330_prep_dma_cyclic()
2701 			src = pch->fifo_dma;  in pl330_prep_dma_cyclic()
2709 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_dma_cyclic()
2710 		desc->rqcfg.brst_len = pch->burst_len;  in pl330_prep_dma_cyclic()
2725 	pch->cyclic = true;  in pl330_prep_dma_cyclic()
2736 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_dma_memcpy()  local
2740 	if (unlikely(!pch || !len))  in pl330_prep_dma_memcpy()
2743 	pl330 = pch->dmac;  in pl330_prep_dma_memcpy()
2745 	desc = __pl330_prep_dma_memcpy(pch, dst, src, len);  in pl330_prep_dma_memcpy()
2811 	struct dma_pl330_chan *pch = to_pchan(chan);  in pl330_prep_slave_sg()  local
2815 	if (unlikely(!pch || !sgl || !sg_len))  in pl330_prep_slave_sg()
2818 	if (!pl330_prep_slave_fifo(pch, direction))  in pl330_prep_slave_sg()
2825 		desc = pl330_get_desc(pch);  in pl330_prep_slave_sg()
2827 			struct pl330_dmac *pl330 = pch->dmac;  in pl330_prep_slave_sg()
2829 			dev_err(pch->dmac->ddma.dev,  in pl330_prep_slave_sg()
2845 			fill_px(&desc->px, pch->fifo_dma, sg_dma_address(sg),  in pl330_prep_slave_sg()
2850 			fill_px(&desc->px, sg_dma_address(sg), pch->fifo_dma,  in pl330_prep_slave_sg()
2854 		desc->rqcfg.brst_size = pch->burst_sz;  in pl330_prep_slave_sg()
2855 		desc->rqcfg.brst_len = pch->burst_len;  in pl330_prep_slave_sg()
2925 	struct dma_pl330_chan *pch, *_p;  in pl330_probe()  local
2993 	pl330->peripherals = kcalloc(num_chan, sizeof(*pch), GFP_KERNEL);  in pl330_probe()
3000 		pch = &pl330->peripherals[i];  in pl330_probe()
3002 		pch->chan.private = adev->dev.of_node;  in pl330_probe()
3003 		INIT_LIST_HEAD(&pch->submitted_list);  in pl330_probe()
3004 		INIT_LIST_HEAD(&pch->work_list);  in pl330_probe()
3005 		INIT_LIST_HEAD(&pch->completed_list);  in pl330_probe()
3006 		spin_lock_init(&pch->lock);  in pl330_probe()
3007 		pch->thread = NULL;  in pl330_probe()
3008 		pch->chan.device = pd;  in pl330_probe()
3009 		pch->dmac = pl330;  in pl330_probe()
3010 		pch->dir = DMA_NONE;  in pl330_probe()
3013 		list_add_tail(&pch->chan.device_node, &pd->channels);  in pl330_probe()
3082 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_probe()
3086 		list_del(&pch->chan.device_node);  in pl330_probe()
3089 		if (pch->thread) {  in pl330_probe()
3090 			pl330_terminate_all(&pch->chan);  in pl330_probe()
3091 			pl330_free_chan_resources(&pch->chan);  in pl330_probe()
3103 	struct dma_pl330_chan *pch, *_p;  in pl330_remove()  local
3120 	list_for_each_entry_safe(pch, _p, &pl330->ddma.channels,  in pl330_remove()
3124 		list_del(&pch->chan.device_node);  in pl330_remove()
3127 		if (pch->thread) {  in pl330_remove()
3128 			pl330_terminate_all(&pch->chan);  in pl330_remove()
3129 			pl330_free_chan_resources(&pch->chan);  in pl330_remove()