319 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))  argument
320 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))  argument
342 static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_cyclic()  argument
344 	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_chan_is_cyclic()
347 static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused()  argument
349 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_chan_is_paused()
352 static inline int at_xdmac_chan_is_paused_internal(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused_internal()  argument
354 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status);  in at_xdmac_chan_is_paused_internal()
373 static void at_xdmac_runtime_suspend_descriptors(struct at_xdmac_chan *atchan)  in at_xdmac_runtime_suspend_descriptors()  argument
375 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_runtime_suspend_descriptors()
378 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) {  in at_xdmac_runtime_suspend_descriptors()
387 static int at_xdmac_runtime_resume_descriptors(struct at_xdmac_chan *atchan)  in at_xdmac_runtime_resume_descriptors()  argument
389 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_runtime_resume_descriptors()
393 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) {  in at_xdmac_runtime_resume_descriptors()
405 static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_enabled()  argument
407 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_chan_is_enabled()
414 	ret = !!(at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask);  in at_xdmac_chan_is_enabled()
425 	struct at_xdmac_chan	*atchan;  in at_xdmac_off()  local
444 			atchan = to_at_xdmac_chan(chan);  in at_xdmac_off()
445 			at_xdmac_runtime_suspend_descriptors(atchan);  in at_xdmac_off()
454 static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_start_xfer()  argument
457 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
465 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
473 		reg |= AT_XDMAC_CNDA_NDAIF(atchan->memif);  in at_xdmac_start_xfer()
475 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);  in at_xdmac_start_xfer()
482 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
495 	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);  in at_xdmac_start_xfer()
500 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);  in at_xdmac_start_xfer()
502 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
504 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
505 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
506 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
507 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
508 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
509 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
511 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);  in at_xdmac_start_xfer()
523 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
524 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
527 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
529 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
530 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
531 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);  in at_xdmac_start_xfer()
533 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
535 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
537 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
538 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
539 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
540 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
541 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
542 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
548 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()  local
552 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
555 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);  in at_xdmac_tx_submit()
556 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
559 		 __func__, atchan, desc);  in at_xdmac_tx_submit()
592 static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)  in at_xdmac_get_desc()  argument
596 	if (list_empty(&atchan->free_descs_list)) {  in at_xdmac_get_desc()
597 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
599 		desc = list_first_entry(&atchan->free_descs_list,  in at_xdmac_get_desc()
639 	struct at_xdmac_chan	*atchan;  in at_xdmac_xlate()  local
654 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
655 	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);  in at_xdmac_xlate()
656 	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);  in at_xdmac_xlate()
657 	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);  in at_xdmac_xlate()
659 		 atchan->memif, atchan->perif, atchan->perid);  in at_xdmac_xlate()
667 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()  local
668 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_compute_chan_conf()
672 		atchan->cfg =  in at_xdmac_compute_chan_conf()
673 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
681 			atchan->cfg |= AT_XDMAC_CC_DIF(atchan->memif) |  in at_xdmac_compute_chan_conf()
682 				       AT_XDMAC_CC_SIF(atchan->perif);  in at_xdmac_compute_chan_conf()
684 		csize = ffs(atchan->sconfig.src_maxburst) - 1;  in at_xdmac_compute_chan_conf()
689 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
690 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;  in at_xdmac_compute_chan_conf()
695 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
697 		atchan->cfg =  in at_xdmac_compute_chan_conf()
698 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
706 			atchan->cfg |= AT_XDMAC_CC_DIF(atchan->perif) |  in at_xdmac_compute_chan_conf()
707 				       AT_XDMAC_CC_SIF(atchan->memif);  in at_xdmac_compute_chan_conf()
709 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;  in at_xdmac_compute_chan_conf()
714 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
715 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;  in at_xdmac_compute_chan_conf()
720 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
723 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
749 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()  local
756 	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));  in at_xdmac_set_slave_config()
766 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()  local
788 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
807 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_slave_sg()
812 						      &atchan->free_descs_list);  in at_xdmac_prep_slave_sg()
818 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_slave_sg()
822 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_slave_sg()
824 		dwidth = at_xdmac_get_dwidth(atchan->cfg);  in at_xdmac_prep_slave_sg()
832 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |  in at_xdmac_prep_slave_sg()
859 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
869 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()  local
884 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {  in at_xdmac_prep_dma_cyclic()
895 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
896 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_cyclic()
901 						      &atchan->free_descs_list);  in at_xdmac_prep_dma_cyclic()
902 			spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
905 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
911 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_dma_cyclic()
915 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_dma_cyclic()
917 		desc->lld.mbr_cfg = atchan->cfg;  in at_xdmac_prep_dma_cyclic()
979 				struct at_xdmac_chan *atchan,  in at_xdmac_interleaved_queue_desc()  argument
1036 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
1037 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_interleaved_queue_desc()
1038 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
1076 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()  local
1101 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1132 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1139 							      &atchan->free_descs_list);  in at_xdmac_prep_interleaved()
1172 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()  local
1213 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1214 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_memcpy()
1215 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1220 						      &atchan->free_descs_list);  in at_xdmac_prep_dma_memcpy()
1276 							 struct at_xdmac_chan *atchan,  in at_xdmac_memset_create_desc()  argument
1318 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1319 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_memset_create_desc()
1320 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1356 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()  local
1365 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1380 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()  local
1398 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1404 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1453 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1516 				      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1541 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()  local
1542 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1560 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_tx_status()
1562 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_tx_status()
1590 		at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);  in at_xdmac_tx_status()
1591 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1622 		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1624 		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1626 		initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);  in at_xdmac_tx_status()
1628 		cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1648 		at_xdmac_write(atxdmac, atxdmac->layout->gswf, atchan->mask);  in at_xdmac_tx_status()
1649 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1676 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_tx_status()
1682 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)  in at_xdmac_advance_work()  argument
1690 	if (at_xdmac_chan_is_enabled(atchan) || list_empty(&atchan->xfers_list))  in at_xdmac_advance_work()
1693 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,  in at_xdmac_advance_work()
1695 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1697 		at_xdmac_start_xfer(atchan, desc);  in at_xdmac_advance_work()
1700 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_handle_cyclic()  argument
1705 	spin_lock_irq(&atchan->lock);  in at_xdmac_handle_cyclic()
1706 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",  in at_xdmac_handle_cyclic()
1707 		__func__, atchan->irq_status);  in at_xdmac_handle_cyclic()
1708 	if (list_empty(&atchan->xfers_list)) {  in at_xdmac_handle_cyclic()
1709 		spin_unlock_irq(&atchan->lock);  in at_xdmac_handle_cyclic()
1712 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,  in at_xdmac_handle_cyclic()
1714 	spin_unlock_irq(&atchan->lock);  in at_xdmac_handle_cyclic()
1721 static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)  in at_xdmac_handle_error()  argument
1723 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()
1737 	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)  in at_xdmac_handle_error()
1738 		dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_handle_error()
1739 	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)  in at_xdmac_handle_error()
1740 		dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_handle_error()
1741 	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)  in at_xdmac_handle_error()
1742 		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_handle_error()
1745 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1746 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1749 	bad_desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_handle_error()
1754 	dev_dbg(chan2dev(&atchan->chan),  in at_xdmac_handle_error()
1767 	struct at_xdmac_chan	*atchan = from_tasklet(atchan, t, tasklet);  in at_xdmac_tasklet()  local
1768 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tasklet()
1773 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_tasklet()
1774 		return at_xdmac_handle_cyclic(atchan);  in at_xdmac_tasklet()
1779 	spin_lock_irq(&atchan->lock);  in at_xdmac_tasklet()
1781 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",  in at_xdmac_tasklet()
1782 		__func__, atchan->irq_status);  in at_xdmac_tasklet()
1784 	if (!(atchan->irq_status & AT_XDMAC_CIS_LIS) &&  in at_xdmac_tasklet()
1785 	    !(atchan->irq_status & error_mask)) {  in at_xdmac_tasklet()
1786 		spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1790 	if (atchan->irq_status & error_mask)  in at_xdmac_tasklet()
1791 		at_xdmac_handle_error(atchan);  in at_xdmac_tasklet()
1793 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc,  in at_xdmac_tasklet()
1795 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1797 		dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");  in at_xdmac_tasklet()
1798 		spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1806 	spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1813 	spin_lock_irq(&atchan->lock);  in at_xdmac_tasklet()
1815 	list_splice_tail_init(&desc->descs_list, &atchan->free_descs_list);  in at_xdmac_tasklet()
1816 	at_xdmac_advance_work(atchan);  in at_xdmac_tasklet()
1817 	spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1830 	struct at_xdmac_chan	*atchan;  in at_xdmac_interrupt()  local
1852 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1853 			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in at_xdmac_interrupt()
1854 			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);  in at_xdmac_interrupt()
1855 			atchan->irq_status = chan_status & chan_imr;  in at_xdmac_interrupt()
1859 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1862 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_interrupt()
1863 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_interrupt()
1864 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_interrupt()
1865 				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_interrupt()
1866 				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_interrupt()
1867 				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_interrupt()
1869 			if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))  in at_xdmac_interrupt()
1870 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1872 			tasklet_schedule(&atchan->tasklet);  in at_xdmac_interrupt()
1883 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()  local
1886 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1888 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_issue_pending()
1889 	at_xdmac_advance_work(atchan);  in at_xdmac_issue_pending()
1890 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_issue_pending()
1898 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()  local
1904 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_config()
1906 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_config()
1912 				      struct at_xdmac_chan *atchan)  in at_xdmac_device_pause_set()  argument
1914 	at_xdmac_write(atxdmac, atxdmac->layout->grws, atchan->mask);  in at_xdmac_device_pause_set()
1915 	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC) &  in at_xdmac_device_pause_set()
1920 static void at_xdmac_device_pause_internal(struct at_xdmac_chan *atchan)  in at_xdmac_device_pause_internal()  argument
1922 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause_internal()
1925 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause_internal()
1926 	set_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status);  in at_xdmac_device_pause_internal()
1927 	at_xdmac_device_pause_set(atxdmac, atchan);  in at_xdmac_device_pause_internal()
1928 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause_internal()
1933 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()  local
1934 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1940 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))  in at_xdmac_device_pause()
1947 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause()
1949 	at_xdmac_device_pause_set(atxdmac, atchan);  in at_xdmac_device_pause()
1951 	at_xdmac_runtime_suspend_descriptors(atchan);  in at_xdmac_device_pause()
1953 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause()
1961 static void at_xdmac_device_resume_internal(struct at_xdmac_chan *atchan)  in at_xdmac_device_resume_internal()  argument
1963 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume_internal()
1966 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume_internal()
1967 	at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask);  in at_xdmac_device_resume_internal()
1968 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED_INTERNAL, &atchan->status);  in at_xdmac_device_resume_internal()
1969 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume_internal()
1974 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()  local
1975 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1985 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume()
1986 	if (!at_xdmac_chan_is_paused(atchan))  in at_xdmac_device_resume()
1990 	ret = at_xdmac_runtime_resume_descriptors(atchan);  in at_xdmac_device_resume()
1994 	at_xdmac_write(atxdmac, atxdmac->layout->grwr, atchan->mask);  in at_xdmac_device_resume()
1995 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_resume()
1998 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
2008 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()  local
2009 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
2019 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
2020 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
2021 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
2025 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node) {  in at_xdmac_device_terminate_all()
2028 				      &atchan->free_descs_list);  in at_xdmac_device_terminate_all()
2040 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_terminate_all()
2041 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_device_terminate_all()
2042 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
2052 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()  local
2056 	if (at_xdmac_chan_is_enabled(atchan)) {  in at_xdmac_alloc_chan_resources()
2062 	if (!list_empty(&atchan->free_descs_list)) {  in at_xdmac_alloc_chan_resources()
2080 		list_add_tail(&desc->desc_node, &atchan->free_descs_list);  in at_xdmac_alloc_chan_resources()
2092 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()  local
2096 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {  in at_xdmac_free_chan_resources()
2135 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()  local
2138 		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))  in atmel_xdmac_prepare()
2155 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()  local
2157 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);  in atmel_xdmac_suspend()
2158 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_suspend()
2159 			if (!at_xdmac_chan_is_paused(atchan)) {  in atmel_xdmac_suspend()
2162 				at_xdmac_device_pause_internal(atchan);  in atmel_xdmac_suspend()
2163 				at_xdmac_runtime_suspend_descriptors(atchan);  in atmel_xdmac_suspend()
2165 			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in atmel_xdmac_suspend()
2166 			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);  in atmel_xdmac_suspend()
2167 			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);  in atmel_xdmac_suspend()
2184 	struct at_xdmac_chan	*atchan;  in atmel_xdmac_resume()  local
2199 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
2200 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in atmel_xdmac_resume()
2206 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
2208 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);  in atmel_xdmac_resume()
2209 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_resume()
2214 			if (at_xdmac_chan_is_paused_internal(atchan)) {  in atmel_xdmac_resume()
2215 				ret = at_xdmac_runtime_resume_descriptors(atchan);  in atmel_xdmac_resume()
2218 				at_xdmac_device_resume_internal(atchan);  in atmel_xdmac_resume()
2226 			else if (at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_resume()
2227 				at_xdmac_device_pause_set(atxdmac, atchan);  in atmel_xdmac_resume()
2229 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);  in atmel_xdmac_resume()
2230 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);  in atmel_xdmac_resume()
2231 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);  in atmel_xdmac_resume()
2233 			if (atxdmac->save_gs & atchan->mask)  in atmel_xdmac_resume()
2234 				at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
2378 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()  local
2380 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2381 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
2384 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2385 		atchan->mask = 1 << i;  in at_xdmac_probe()
2387 		spin_lock_init(&atchan->lock);  in at_xdmac_probe()
2388 		INIT_LIST_HEAD(&atchan->xfers_list);  in at_xdmac_probe()
2389 		INIT_LIST_HEAD(&atchan->free_descs_list);  in at_xdmac_probe()
2390 		tasklet_setup(&atchan->tasklet, at_xdmac_tasklet);  in at_xdmac_probe()
2393 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in at_xdmac_probe()
2450 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()  local
2452 		tasklet_kill(&atchan->tasklet);  in at_xdmac_remove()
2453 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()