231 	struct sprd_dma_chn *schan = to_sprd_dma_chan(c);  in to_sprd_dma_dev()  local
233 	return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);  in to_sprd_dma_dev()
251 static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,  in sprd_dma_chn_update()  argument
254 	u32 orig = readl(schan->chn_base + reg);  in sprd_dma_chn_update()
258 	writel(tmp, schan->chn_base + reg);  in sprd_dma_chn_update()
290 static void sprd_dma_set_uid(struct sprd_dma_chn *schan)  in sprd_dma_set_uid()  argument
292 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_set_uid()
293 	u32 dev_id = schan->dev_id;  in sprd_dma_set_uid()
299 		writel(schan->chn_num + 1, sdev->glb_base + uid_offset);  in sprd_dma_set_uid()
303 static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)  in sprd_dma_unset_uid()  argument
305 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_unset_uid()
306 	u32 dev_id = schan->dev_id;  in sprd_dma_unset_uid()
316 static void sprd_dma_clear_int(struct sprd_dma_chn *schan)  in sprd_dma_clear_int()  argument
318 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,  in sprd_dma_clear_int()
323 static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)  in sprd_dma_enable_chn()  argument
325 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,  in sprd_dma_enable_chn()
329 static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)  in sprd_dma_disable_chn()  argument
331 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);  in sprd_dma_disable_chn()
334 static void sprd_dma_soft_request(struct sprd_dma_chn *schan)  in sprd_dma_soft_request()  argument
336 	sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,  in sprd_dma_soft_request()
340 static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)  in sprd_dma_pause_resume()  argument
342 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_pause_resume()
346 		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,  in sprd_dma_pause_resume()
350 			pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);  in sprd_dma_pause_resume()
361 		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,  in sprd_dma_pause_resume()
366 static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)  in sprd_dma_stop_and_disable()  argument
368 	u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);  in sprd_dma_stop_and_disable()
373 	sprd_dma_pause_resume(schan, true);  in sprd_dma_stop_and_disable()
374 	sprd_dma_disable_chn(schan);  in sprd_dma_stop_and_disable()
377 static unsigned long sprd_dma_get_src_addr(struct sprd_dma_chn *schan)  in sprd_dma_get_src_addr()  argument
381 	addr = readl(schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);  in sprd_dma_get_src_addr()
382 	addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_PTR) &  in sprd_dma_get_src_addr()
388 static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)  in sprd_dma_get_dst_addr()  argument
392 	addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);  in sprd_dma_get_dst_addr()
393 	addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &  in sprd_dma_get_dst_addr()
399 static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)  in sprd_dma_get_int_type()  argument
401 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_get_int_type()
402 	u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &  in sprd_dma_get_int_type()
427 static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)  in sprd_dma_get_req_type()  argument
429 	u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);  in sprd_dma_get_req_type()
434 static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan)  in sprd_dma_set_2stage_config()  argument
436 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_set_2stage_config()
437 	u32 val, chn = schan->chn_num + 1;  in sprd_dma_set_2stage_config()
439 	switch (schan->chn_mode) {  in sprd_dma_set_2stage_config()
442 		val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;  in sprd_dma_set_2stage_config()
444 		if (schan->int_type != SPRD_DMA_NO_INT)  in sprd_dma_set_2stage_config()
452 		val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;  in sprd_dma_set_2stage_config()
454 		if (schan->int_type != SPRD_DMA_NO_INT)  in sprd_dma_set_2stage_config()
464 		if (schan->int_type != SPRD_DMA_NO_INT)  in sprd_dma_set_2stage_config()
474 		if (schan->int_type != SPRD_DMA_NO_INT)  in sprd_dma_set_2stage_config()
482 			schan->chn_mode);  in sprd_dma_set_2stage_config()
489 static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable)  in sprd_dma_set_pending()  argument
491 	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);  in sprd_dma_set_pending()
494 	if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)  in sprd_dma_set_pending()
498 	req_id = schan->dev_id - 1;  in sprd_dma_set_pending()
511 static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,  in sprd_dma_set_chn_config()  argument
516 	writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);  in sprd_dma_set_chn_config()
517 	writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);  in sprd_dma_set_chn_config()
518 	writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);  in sprd_dma_set_chn_config()
519 	writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);  in sprd_dma_set_chn_config()
520 	writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);  in sprd_dma_set_chn_config()
521 	writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);  in sprd_dma_set_chn_config()
522 	writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);  in sprd_dma_set_chn_config()
523 	writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);  in sprd_dma_set_chn_config()
524 	writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);  in sprd_dma_set_chn_config()
525 	writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);  in sprd_dma_set_chn_config()
526 	writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);  in sprd_dma_set_chn_config()
527 	writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);  in sprd_dma_set_chn_config()
528 	writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);  in sprd_dma_set_chn_config()
529 	writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);  in sprd_dma_set_chn_config()
530 	writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);  in sprd_dma_set_chn_config()
531 	writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);  in sprd_dma_set_chn_config()
534 static void sprd_dma_start(struct sprd_dma_chn *schan)  in sprd_dma_start()  argument
536 	struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);  in sprd_dma_start()
542 	schan->cur_desc = to_sprd_dma_desc(vd);  in sprd_dma_start()
548 	if (schan->chn_mode && sprd_dma_set_2stage_config(schan))  in sprd_dma_start()
555 	sprd_dma_set_chn_config(schan, schan->cur_desc);  in sprd_dma_start()
556 	sprd_dma_set_uid(schan);  in sprd_dma_start()
557 	sprd_dma_set_pending(schan, true);  in sprd_dma_start()
558 	sprd_dma_enable_chn(schan);  in sprd_dma_start()
560 	if (schan->dev_id == SPRD_DMA_SOFTWARE_UID &&  in sprd_dma_start()
561 	    schan->chn_mode != SPRD_DMA_DST_CHN0 &&  in sprd_dma_start()
562 	    schan->chn_mode != SPRD_DMA_DST_CHN1)  in sprd_dma_start()
563 		sprd_dma_soft_request(schan);  in sprd_dma_start()
566 static void sprd_dma_stop(struct sprd_dma_chn *schan)  in sprd_dma_stop()  argument
568 	sprd_dma_stop_and_disable(schan);  in sprd_dma_stop()
569 	sprd_dma_set_pending(schan, false);  in sprd_dma_stop()
570 	sprd_dma_unset_uid(schan);  in sprd_dma_stop()
571 	sprd_dma_clear_int(schan);  in sprd_dma_stop()
572 	schan->cur_desc = NULL;  in sprd_dma_stop()
592 	struct sprd_dma_chn *schan;  in dma_irq_handle()  local
602 		schan = &sdev->channels[i];  in dma_irq_handle()
604 		spin_lock(&schan->vc.lock);  in dma_irq_handle()
606 		sdesc = schan->cur_desc;  in dma_irq_handle()
608 			spin_unlock(&schan->vc.lock);  in dma_irq_handle()
612 		int_type = sprd_dma_get_int_type(schan);  in dma_irq_handle()
613 		req_type = sprd_dma_get_req_type(schan);  in dma_irq_handle()
614 		sprd_dma_clear_int(schan);  in dma_irq_handle()
617 		cyclic = schan->linklist.phy_addr ? true : false;  in dma_irq_handle()
626 				schan->cur_desc = NULL;  in dma_irq_handle()
627 				sprd_dma_start(schan);  in dma_irq_handle()
630 		spin_unlock(&schan->vc.lock);  in dma_irq_handle()
643 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_free_chan_resources()  local
647 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_free_chan_resources()
648 	if (schan->cur_desc)  in sprd_dma_free_chan_resources()
649 		cur_vd = &schan->cur_desc->vd;  in sprd_dma_free_chan_resources()
651 	sprd_dma_stop(schan);  in sprd_dma_free_chan_resources()
652 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_free_chan_resources()
657 	vchan_free_chan_resources(&schan->vc);  in sprd_dma_free_chan_resources()
665 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_tx_status()  local
675 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_tx_status()
676 	vd = vchan_find_desc(&schan->vc, cookie);  in sprd_dma_tx_status()
689 	} else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {  in sprd_dma_tx_status()
690 		struct sprd_dma_desc *sdesc = schan->cur_desc;  in sprd_dma_tx_status()
693 			pos = sprd_dma_get_dst_addr(schan);  in sprd_dma_tx_status()
695 			pos = sprd_dma_get_src_addr(schan);  in sprd_dma_tx_status()
699 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_tx_status()
707 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_issue_pending()  local
710 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_issue_pending()
711 	if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)  in sprd_dma_issue_pending()
712 		sprd_dma_start(schan);  in sprd_dma_issue_pending()
713 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_issue_pending()
753 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_fill_desc()  local
754 	enum sprd_dma_chn_mode chn_mode = schan->chn_mode;  in sprd_dma_fill_desc()
831 	temp |= schan->linklist.wrap_addr ?  in sprd_dma_fill_desc()
844 	if (schan->linklist.phy_addr) {  in sprd_dma_fill_desc()
856 		llist_ptr = schan->linklist.phy_addr + temp;  in sprd_dma_fill_desc()
861 		if (schan->linklist.wrap_addr) {  in sprd_dma_fill_desc()
862 			hw->wrap_ptr |= schan->linklist.wrap_addr &  in sprd_dma_fill_desc()
883 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_fill_linklist_desc()  local
886 	if (!schan->linklist.virt_addr)  in sprd_dma_fill_linklist_desc()
889 	hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr +  in sprd_dma_fill_linklist_desc()
900 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_prep_dma_memcpy()  local
948 	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);  in sprd_dma_prep_dma_memcpy()
956 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_prep_slave_sg()  local
957 	struct dma_slave_config *slave_cfg = &schan->slave_cfg;  in sprd_dma_prep_slave_sg()
972 		schan->linklist.phy_addr = ll_cfg->phy_addr;  in sprd_dma_prep_slave_sg()
973 		schan->linklist.virt_addr = ll_cfg->virt_addr;  in sprd_dma_prep_slave_sg()
974 		schan->linklist.wrap_addr = ll_cfg->wrap_addr;  in sprd_dma_prep_slave_sg()
976 		schan->linklist.phy_addr = 0;  in sprd_dma_prep_slave_sg()
977 		schan->linklist.virt_addr = 0;  in sprd_dma_prep_slave_sg()
978 		schan->linklist.wrap_addr = 0;  in sprd_dma_prep_slave_sg()
985 	schan->chn_mode =  in sprd_dma_prep_slave_sg()
987 	schan->trg_mode =  in sprd_dma_prep_slave_sg()
989 	schan->int_type = flags & SPRD_DMA_INT_TYPE_MASK;  in sprd_dma_prep_slave_sg()
1036 	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);  in sprd_dma_prep_slave_sg()
1042 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_slave_config()  local
1043 	struct dma_slave_config *slave_cfg = &schan->slave_cfg;  in sprd_dma_slave_config()
1051 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_pause()  local
1054 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_pause()
1055 	sprd_dma_pause_resume(schan, true);  in sprd_dma_pause()
1056 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_pause()
1063 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_resume()  local
1066 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_resume()
1067 	sprd_dma_pause_resume(schan, false);  in sprd_dma_resume()
1068 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_resume()
1075 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_terminate_all()  local
1080 	spin_lock_irqsave(&schan->vc.lock, flags);  in sprd_dma_terminate_all()
1081 	if (schan->cur_desc)  in sprd_dma_terminate_all()
1082 		cur_vd = &schan->cur_desc->vd;  in sprd_dma_terminate_all()
1084 	sprd_dma_stop(schan);  in sprd_dma_terminate_all()
1086 	vchan_get_all_descriptors(&schan->vc, &head);  in sprd_dma_terminate_all()
1087 	spin_unlock_irqrestore(&schan->vc.lock, flags);  in sprd_dma_terminate_all()
1092 	vchan_dma_desc_free_list(&schan->vc, &head);  in sprd_dma_terminate_all()
1105 	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);  in sprd_dma_filter_fn()  local
1108 	schan->dev_id = slave_id;  in sprd_dma_filter_fn()