256 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))  argument
257 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))  argument
279 static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_cyclic()  argument
281 	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_chan_is_cyclic()
284 static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused()  argument
286 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_chan_is_paused()
316 static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_enabled()  argument
318 	return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;  in at_xdmac_chan_is_enabled()
333 static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_start_xfer()  argument
336 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
339 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
341 	if (at_xdmac_chan_is_enabled(atchan))  in at_xdmac_start_xfer()
349 	      | AT_XDMAC_CNDA_NDAIF(atchan->memif);  in at_xdmac_start_xfer()
350 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);  in at_xdmac_start_xfer()
357 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
369 	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);  in at_xdmac_start_xfer()
374 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);  in at_xdmac_start_xfer()
376 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
378 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
379 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
380 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
381 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
382 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
383 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
385 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);  in at_xdmac_start_xfer()
397 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
398 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
401 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
403 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
404 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
405 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);  in at_xdmac_start_xfer()
407 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
409 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
411 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
412 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
413 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
414 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
415 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
416 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
423 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()  local
427 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
431 		 __func__, atchan, desc);  in at_xdmac_tx_submit()
432 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);  in at_xdmac_tx_submit()
433 	if (list_is_singular(&atchan->xfers_list))  in at_xdmac_tx_submit()
434 		at_xdmac_start_xfer(atchan, desc);  in at_xdmac_tx_submit()
436 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
468 static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)  in at_xdmac_get_desc()  argument
472 	if (list_empty(&atchan->free_descs_list)) {  in at_xdmac_get_desc()
473 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
475 		desc = list_first_entry(&atchan->free_descs_list,  in at_xdmac_get_desc()
515 	struct at_xdmac_chan	*atchan;  in at_xdmac_xlate()  local
530 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
531 	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);  in at_xdmac_xlate()
532 	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);  in at_xdmac_xlate()
533 	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);  in at_xdmac_xlate()
535 		 atchan->memif, atchan->perif, atchan->perid);  in at_xdmac_xlate()
543 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()  local
547 		atchan->cfg =  in at_xdmac_compute_chan_conf()
548 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
551 			| AT_XDMAC_CC_DIF(atchan->memif)  in at_xdmac_compute_chan_conf()
552 			| AT_XDMAC_CC_SIF(atchan->perif)  in at_xdmac_compute_chan_conf()
557 		csize = ffs(atchan->sconfig.src_maxburst) - 1;  in at_xdmac_compute_chan_conf()
562 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
563 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;  in at_xdmac_compute_chan_conf()
568 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
570 		atchan->cfg =  in at_xdmac_compute_chan_conf()
571 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
574 			| AT_XDMAC_CC_DIF(atchan->perif)  in at_xdmac_compute_chan_conf()
575 			| AT_XDMAC_CC_SIF(atchan->memif)  in at_xdmac_compute_chan_conf()
580 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;  in at_xdmac_compute_chan_conf()
585 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
586 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;  in at_xdmac_compute_chan_conf()
591 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
594 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
620 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()  local
627 	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));  in at_xdmac_set_slave_config()
637 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()  local
659 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
678 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_slave_sg()
682 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_slave_sg()
688 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_slave_sg()
692 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_slave_sg()
694 		dwidth = at_xdmac_get_dwidth(atchan->cfg);  in at_xdmac_prep_slave_sg()
702 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |  in at_xdmac_prep_slave_sg()
729 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
739 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()  local
754 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {  in at_xdmac_prep_dma_cyclic()
765 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
766 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_cyclic()
770 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_cyclic()
771 			spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
774 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
780 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_dma_cyclic()
784 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_dma_cyclic()
786 		desc->lld.mbr_cfg = atchan->cfg;  in at_xdmac_prep_dma_cyclic()
848 				struct at_xdmac_chan *atchan,  in at_xdmac_interleaved_queue_desc()  argument
903 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
904 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_interleaved_queue_desc()
905 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
943 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()  local
968 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
997 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1003 						 &atchan->free_descs_list);  in at_xdmac_prep_interleaved()
1036 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()  local
1075 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1076 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_memcpy()
1077 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1081 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_memcpy()
1137 							 struct at_xdmac_chan *atchan,  in at_xdmac_memset_create_desc()  argument
1176 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1177 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_memset_create_desc()
1178 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1208 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()  local
1217 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1232 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()  local
1250 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1256 					 &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1305 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1368 				      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1393 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()  local
1394 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1411 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_tx_status()
1413 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_tx_status()
1441 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1442 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1473 		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1475 		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1477 		initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);  in at_xdmac_tx_status()
1479 		cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1499 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1500 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1525 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_tx_status()
1530 static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_remove_xfer()  argument
1533 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1540 	list_splice_init(&desc->descs_list, &atchan->free_descs_list);  in at_xdmac_remove_xfer()
1543 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)  in at_xdmac_advance_work()  argument
1551 	if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {  in at_xdmac_advance_work()
1552 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_advance_work()
1555 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1557 			at_xdmac_start_xfer(atchan, desc);  in at_xdmac_advance_work()
1561 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_handle_cyclic()  argument
1566 	if (!list_empty(&atchan->xfers_list)) {  in at_xdmac_handle_cyclic()
1567 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_handle_cyclic()
1576 static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)  in at_xdmac_handle_error()  argument
1578 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()
1587 	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)  in at_xdmac_handle_error()
1588 		dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_handle_error()
1589 	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)  in at_xdmac_handle_error()
1590 		dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_handle_error()
1591 	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)  in at_xdmac_handle_error()
1592 		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_handle_error()
1594 	spin_lock_irq(&atchan->lock);  in at_xdmac_handle_error()
1597 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1598 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1601 	bad_desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_handle_error()
1605 	spin_unlock_irq(&atchan->lock);  in at_xdmac_handle_error()
1608 	dev_dbg(chan2dev(&atchan->chan),  in at_xdmac_handle_error()
1618 	struct at_xdmac_chan	*atchan = from_tasklet(atchan, t, tasklet);  in at_xdmac_tasklet()  local
1622 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",  in at_xdmac_tasklet()
1623 		__func__, atchan->irq_status);  in at_xdmac_tasklet()
1629 	if (at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1630 		at_xdmac_handle_cyclic(atchan);  in at_xdmac_tasklet()
1631 	} else if ((atchan->irq_status & AT_XDMAC_CIS_LIS)  in at_xdmac_tasklet()
1632 		   || (atchan->irq_status & error_mask)) {  in at_xdmac_tasklet()
1635 		if (atchan->irq_status & error_mask)  in at_xdmac_tasklet()
1636 			at_xdmac_handle_error(atchan);  in at_xdmac_tasklet()
1638 		spin_lock_irq(&atchan->lock);  in at_xdmac_tasklet()
1639 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_tasklet()
1642 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1644 			dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");  in at_xdmac_tasklet()
1645 			spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1651 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_tasklet()
1652 		spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1660 		spin_lock_irq(&atchan->lock);  in at_xdmac_tasklet()
1661 		at_xdmac_advance_work(atchan);  in at_xdmac_tasklet()
1662 		spin_unlock_irq(&atchan->lock);  in at_xdmac_tasklet()
1669 	struct at_xdmac_chan	*atchan;  in at_xdmac_interrupt()  local
1691 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1692 			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in at_xdmac_interrupt()
1693 			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);  in at_xdmac_interrupt()
1694 			atchan->irq_status = chan_status & chan_imr;  in at_xdmac_interrupt()
1698 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1701 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_interrupt()
1702 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_interrupt()
1703 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_interrupt()
1704 				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_interrupt()
1705 				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_interrupt()
1706 				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_interrupt()
1708 			if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))  in at_xdmac_interrupt()
1709 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1711 			tasklet_schedule(&atchan->tasklet);  in at_xdmac_interrupt()
1722 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()  local
1725 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1727 	if (!at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_issue_pending()
1728 		spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_issue_pending()
1729 		at_xdmac_advance_work(atchan);  in at_xdmac_issue_pending()
1730 		spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_issue_pending()
1739 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()  local
1745 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_config()
1747 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_config()
1754 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()  local
1755 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1760 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))  in at_xdmac_device_pause()
1763 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause()
1764 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1765 	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)  in at_xdmac_device_pause()
1768 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause()
1775 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()  local
1776 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1781 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume()
1782 	if (!at_xdmac_chan_is_paused(atchan)) {  in at_xdmac_device_resume()
1783 		spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1787 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1788 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_resume()
1789 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1797 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()  local
1798 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1803 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1804 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1805 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1809 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)  in at_xdmac_device_terminate_all()
1810 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_device_terminate_all()
1812 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_terminate_all()
1813 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_device_terminate_all()
1814 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1821 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()  local
1825 	if (at_xdmac_chan_is_enabled(atchan)) {  in at_xdmac_alloc_chan_resources()
1831 	if (!list_empty(&atchan->free_descs_list)) {  in at_xdmac_alloc_chan_resources()
1844 		list_add_tail(&desc->desc_node, &atchan->free_descs_list);  in at_xdmac_alloc_chan_resources()
1856 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()  local
1860 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {  in at_xdmac_free_chan_resources()
1876 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()  local
1879 		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))  in atmel_xdmac_prepare()
1895 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()  local
1897 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);  in atmel_xdmac_suspend()
1898 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_suspend()
1899 			if (!at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_suspend()
1901 			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in atmel_xdmac_suspend()
1902 			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);  in atmel_xdmac_suspend()
1903 			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);  in atmel_xdmac_suspend()
1916 	struct at_xdmac_chan	*atchan;  in atmel_xdmac_resume()  local
1927 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1928 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in atmel_xdmac_resume()
1934 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1935 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);  in atmel_xdmac_resume()
1936 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_resume()
1937 			if (at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_resume()
1939 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);  in atmel_xdmac_resume()
1940 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);  in atmel_xdmac_resume()
1941 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);  in atmel_xdmac_resume()
1943 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
2054 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()  local
2056 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2057 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
2060 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2061 		atchan->mask = 1 << i;  in at_xdmac_probe()
2063 		spin_lock_init(&atchan->lock);  in at_xdmac_probe()
2064 		INIT_LIST_HEAD(&atchan->xfers_list);  in at_xdmac_probe()
2065 		INIT_LIST_HEAD(&atchan->free_descs_list);  in at_xdmac_probe()
2066 		tasklet_setup(&atchan->tasklet, at_xdmac_tasklet);  in at_xdmac_probe()
2069 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in at_xdmac_probe()
2114 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()  local
2116 		tasklet_kill(&atchan->tasklet);  in at_xdmac_remove()
2117 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()