196 	struct sprd_dma_chn *schan = to_sprd_dma_chan(c);  in to_sprd_dma_dev()  local
198 	return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);  in to_sprd_dma_dev()
206 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,  in sprd_dma_chn_update()  argument
209 	u32 orig = readl(schan->chn_base + reg);  in sprd_dma_chn_update()
213 	writel(tmp, schan->chn_base + reg);  in sprd_dma_chn_update()
245 static void sprd_dma_set_uid(struct sprd_dma_chn *schan)  in sprd_dma_set_uid()  argument
247 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_set_uid()
248 	u32 dev_id = schan->dev_id;  in sprd_dma_set_uid()
254 		writel(schan->chn_num + 1, sdev->glb_base + uid_offset);  in sprd_dma_set_uid()
258 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)  in sprd_dma_unset_uid()  argument
260 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_unset_uid()
261 	u32 dev_id = schan->dev_id;  in sprd_dma_unset_uid()
271 static void sprd_dma_clear_int(struct sprd_dma_chn *schan)  in sprd_dma_clear_int()  argument
273 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,  in sprd_dma_clear_int()
278 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)  in sprd_dma_enable_chn()  argument
280 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,  in sprd_dma_enable_chn()
284 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)  in sprd_dma_disable_chn()  argument
286 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);  in sprd_dma_disable_chn()
289 static void sprd_dma_soft_request(struct sprd_dma_chn *schan)  in sprd_dma_soft_request()  argument
291 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,  in sprd_dma_soft_request()
295 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)  in sprd_dma_pause_resume()  argument
297 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_pause_resume()
301 		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,  in sprd_dma_pause_resume()
305 			pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);  in sprd_dma_pause_resume()
316 		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,  in sprd_dma_pause_resume()
321 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)  in sprd_dma_stop_and_disable()  argument
323 	u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);  in sprd_dma_stop_and_disable()
328 	sprd_dma_pause_resume(schan, true);  in sprd_dma_stop_and_disable()
329 	sprd_dma_disable_chn(schan);  in sprd_dma_stop_and_disable()
332 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)  in sprd_dma_get_dst_addr()  argument
336 	addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);  in sprd_dma_get_dst_addr()
337 	addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &  in sprd_dma_get_dst_addr()
343 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)  in sprd_dma_get_int_type()  argument
345 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_get_int_type()
346 	u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &  in sprd_dma_get_int_type()
371 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)  in sprd_dma_get_req_type()  argument
373 	u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);  in sprd_dma_get_req_type()
378 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,  in sprd_dma_set_chn_config()  argument
383 	writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);  in sprd_dma_set_chn_config()
384 	writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);  in sprd_dma_set_chn_config()
385 	writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);  in sprd_dma_set_chn_config()
386 	writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);  in sprd_dma_set_chn_config()
387 	writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);  in sprd_dma_set_chn_config()
388 	writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);  in sprd_dma_set_chn_config()
389 	writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);  in sprd_dma_set_chn_config()
390 	writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);  in sprd_dma_set_chn_config()
391 	writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);  in sprd_dma_set_chn_config()
392 	writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);  in sprd_dma_set_chn_config()
393 	writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);  in sprd_dma_set_chn_config()
394 	writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);  in sprd_dma_set_chn_config()
395 	writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);  in sprd_dma_set_chn_config()
396 	writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);  in sprd_dma_set_chn_config()
397 	writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);  in sprd_dma_set_chn_config()
398 	writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);  in sprd_dma_set_chn_config()
401 static void sprd_dma_start(struct sprd_dma_chn *schan)  in sprd_dma_start()  argument
403 	struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);  in sprd_dma_start()
409 	schan->cur_desc = to_sprd_dma_desc(vd);  in sprd_dma_start()
415 	sprd_dma_set_chn_config(schan, schan->cur_desc);  in sprd_dma_start()
416 	sprd_dma_set_uid(schan);  in sprd_dma_start()
417 	sprd_dma_enable_chn(schan);  in sprd_dma_start()
419 	if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)  in sprd_dma_start()
420 		sprd_dma_soft_request(schan);  in sprd_dma_start()
423 static void sprd_dma_stop(struct sprd_dma_chn *schan)  in sprd_dma_stop()  argument
425 	sprd_dma_stop_and_disable(schan);  in sprd_dma_stop()
426 	sprd_dma_unset_uid(schan);  in sprd_dma_stop()
427 	sprd_dma_clear_int(schan);  in sprd_dma_stop()
447 	struct sprd_dma_chn *schan;  in dma_irq_handle()  local
457 		schan = &sdev->channels[i];  in dma_irq_handle()
459 		spin_lock(&schan->vc.lock);  in dma_irq_handle()
460 		int_type = sprd_dma_get_int_type(schan);  in dma_irq_handle()
461 		req_type = sprd_dma_get_req_type(schan);  in dma_irq_handle()
462 		sprd_dma_clear_int(schan);  in dma_irq_handle()
464 		sdesc = schan->cur_desc;  in dma_irq_handle()
471 			schan->cur_desc = NULL;  in dma_irq_handle()
472 			sprd_dma_start(schan);  in dma_irq_handle()
474 		spin_unlock(&schan->vc.lock);  in dma_irq_handle()
482 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_alloc_chan_resources()  local
489 	schan->dev_id = SPRD_DMA_SOFTWARE_UID;  in sprd_dma_alloc_chan_resources()
495 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_free_chan_resources()  local
498 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_free_chan_resources()
499 	sprd_dma_stop(schan);  in sprd_dma_free_chan_resources()
500 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_free_chan_resources()
502 	vchan_free_chan_resources(&schan->vc);  in sprd_dma_free_chan_resources()
510 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_tx_status()  local
520 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_tx_status()
521 	vd = vchan_find_desc(&schan->vc, cookie);  in sprd_dma_tx_status()
534 	} else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {  in sprd_dma_tx_status()
535 		pos = sprd_dma_get_dst_addr(schan);  in sprd_dma_tx_status()
539 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_tx_status()
547 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_issue_pending()  local
550 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_issue_pending()
551 	if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)  in sprd_dma_issue_pending()
552 		sprd_dma_start(schan);  in sprd_dma_issue_pending()
553 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_issue_pending()
592 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_fill_desc()  local
628 		schan->dev_id = slave_cfg->slave_id;  in sprd_dma_fill_desc()
683 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_prep_dma_memcpy()  local
731 	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);  in sprd_dma_prep_dma_memcpy()
739 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_prep_slave_sg()  local
740 	struct dma_slave_config *slave_cfg = &schan->slave_cfg;  in sprd_dma_prep_slave_sg()
774 	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);  in sprd_dma_prep_slave_sg()
780 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_slave_config()  local
781 	struct dma_slave_config *slave_cfg = &schan->slave_cfg;  in sprd_dma_slave_config()
792 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_pause()  local
795 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_pause()
796 	sprd_dma_pause_resume(schan, true);  in sprd_dma_pause()
797 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_pause()
804 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_resume()  local
807 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_resume()
808 	sprd_dma_pause_resume(schan, false);  in sprd_dma_resume()
809 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_resume()
816 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_terminate_all()  local
820 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_terminate_all()
821 	sprd_dma_stop(schan);  in sprd_dma_terminate_all()
823 	vchan_get_all_descriptors(&schan->vc, &head);  in sprd_dma_terminate_all()
824 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_terminate_all()
826 	vchan_dma_desc_free_list(&schan->vc, &head);  in sprd_dma_terminate_all()
839 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_filter_fn()  local
840 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_filter_fn()
844 		return req == schan->chn_num + 1;  in sprd_dma_filter_fn()