256 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))  argument
257 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))  argument
279 static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_cyclic()  argument
281 	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_chan_is_cyclic()
284 static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused()  argument
286 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_chan_is_paused()
316 static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_enabled()  argument
318 	return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;  in at_xdmac_chan_is_enabled()
333 static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_start_xfer()  argument
336 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
339 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
341 	if (at_xdmac_chan_is_enabled(atchan))  in at_xdmac_start_xfer()
349 	      | AT_XDMAC_CNDA_NDAIF(atchan->memif);  in at_xdmac_start_xfer()
350 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);  in at_xdmac_start_xfer()
357 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
369 	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);  in at_xdmac_start_xfer()
374 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);  in at_xdmac_start_xfer()
376 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
378 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
379 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
380 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
381 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
382 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
383 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
385 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);  in at_xdmac_start_xfer()
397 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
398 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
401 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
403 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
404 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
405 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);  in at_xdmac_start_xfer()
407 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
409 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
411 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
412 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
413 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
414 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
415 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
416 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
423 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()  local
427 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
431 		 __func__, atchan, desc);  in at_xdmac_tx_submit()
432 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);  in at_xdmac_tx_submit()
433 	if (list_is_singular(&atchan->xfers_list))  in at_xdmac_tx_submit()
434 		at_xdmac_start_xfer(atchan, desc);  in at_xdmac_tx_submit()
436 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
468 static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)  in at_xdmac_get_desc()  argument
472 	if (list_empty(&atchan->free_descs_list)) {  in at_xdmac_get_desc()
473 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
475 		desc = list_first_entry(&atchan->free_descs_list,  in at_xdmac_get_desc()
515 	struct at_xdmac_chan	*atchan;  in at_xdmac_xlate()  local
530 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
531 	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);  in at_xdmac_xlate()
532 	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);  in at_xdmac_xlate()
533 	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);  in at_xdmac_xlate()
535 		 atchan->memif, atchan->perif, atchan->perid);  in at_xdmac_xlate()
543 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()  local
547 		atchan->cfg =  in at_xdmac_compute_chan_conf()
548 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
551 			| AT_XDMAC_CC_DIF(atchan->memif)  in at_xdmac_compute_chan_conf()
552 			| AT_XDMAC_CC_SIF(atchan->perif)  in at_xdmac_compute_chan_conf()
557 		csize = ffs(atchan->sconfig.src_maxburst) - 1;  in at_xdmac_compute_chan_conf()
562 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
563 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;  in at_xdmac_compute_chan_conf()
568 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
570 		atchan->cfg =  in at_xdmac_compute_chan_conf()
571 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
574 			| AT_XDMAC_CC_DIF(atchan->perif)  in at_xdmac_compute_chan_conf()
575 			| AT_XDMAC_CC_SIF(atchan->memif)  in at_xdmac_compute_chan_conf()
580 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;  in at_xdmac_compute_chan_conf()
585 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
586 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;  in at_xdmac_compute_chan_conf()
591 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
594 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
620 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()  local
627 	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));  in at_xdmac_set_slave_config()
637 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()  local
659 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
678 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_slave_sg()
682 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_slave_sg()
688 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_slave_sg()
692 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_slave_sg()
694 		dwidth = at_xdmac_get_dwidth(atchan->cfg);  in at_xdmac_prep_slave_sg()
702 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |  in at_xdmac_prep_slave_sg()
729 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
739 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()  local
754 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {  in at_xdmac_prep_dma_cyclic()
765 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
766 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_cyclic()
770 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_cyclic()
771 			spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
774 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
780 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_dma_cyclic()
784 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_dma_cyclic()
786 		desc->lld.mbr_cfg = atchan->cfg;  in at_xdmac_prep_dma_cyclic()
848 				struct at_xdmac_chan *atchan,  in at_xdmac_interleaved_queue_desc()  argument
903 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
904 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_interleaved_queue_desc()
905 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
943 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()  local
968 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
997 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1003 						 &atchan->free_descs_list);  in at_xdmac_prep_interleaved()
1036 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()  local
1075 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1076 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_memcpy()
1077 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1081 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_memcpy()
1137 							 struct at_xdmac_chan *atchan,  in at_xdmac_memset_create_desc()  argument
1176 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1177 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_memset_create_desc()
1178 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1208 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()  local
1217 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1232 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()  local
1250 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1256 					 &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1305 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1368 				      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1393 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()  local
1394 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1411 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_tx_status()
1413 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_tx_status()
1441 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1442 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1473 		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1475 		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1477 		initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);  in at_xdmac_tx_status()
1479 		cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1499 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1500 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1525 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_tx_status()
1530 static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_remove_xfer()  argument
1533 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1540 	list_splice_init(&desc->descs_list, &atchan->free_descs_list);  in at_xdmac_remove_xfer()
1543 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)  in at_xdmac_advance_work()  argument
1548 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_advance_work()
1554 	if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {  in at_xdmac_advance_work()
1555 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_advance_work()
1558 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1560 			at_xdmac_start_xfer(atchan, desc);  in at_xdmac_advance_work()
1563 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_advance_work()
1566 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_handle_cyclic()  argument
1571 	if (!list_empty(&atchan->xfers_list)) {  in at_xdmac_handle_cyclic()
1572 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_handle_cyclic()
1581 static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)  in at_xdmac_handle_error()  argument
1583 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_handle_error()
1592 	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)  in at_xdmac_handle_error()
1593 		dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_handle_error()
1594 	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)  in at_xdmac_handle_error()
1595 		dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_handle_error()
1596 	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)  in at_xdmac_handle_error()
1597 		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_handle_error()
1599 	spin_lock_bh(&atchan->lock);  in at_xdmac_handle_error()
1602 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_handle_error()
1603 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_handle_error()
1606 	bad_desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_handle_error()
1610 	spin_unlock_bh(&atchan->lock);  in at_xdmac_handle_error()
1613 	dev_dbg(chan2dev(&atchan->chan),  in at_xdmac_handle_error()
1623 	struct at_xdmac_chan	*atchan = (struct at_xdmac_chan *)data;  in at_xdmac_tasklet()  local
1627 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",  in at_xdmac_tasklet()
1628 		__func__, atchan->irq_status);  in at_xdmac_tasklet()
1634 	if (at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1635 		at_xdmac_handle_cyclic(atchan);  in at_xdmac_tasklet()
1636 	} else if ((atchan->irq_status & AT_XDMAC_CIS_LIS)  in at_xdmac_tasklet()
1637 		   || (atchan->irq_status & error_mask)) {  in at_xdmac_tasklet()
1640 		if (atchan->irq_status & error_mask)  in at_xdmac_tasklet()
1641 			at_xdmac_handle_error(atchan);  in at_xdmac_tasklet()
1643 		spin_lock(&atchan->lock);  in at_xdmac_tasklet()
1644 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_tasklet()
1647 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1649 			dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");  in at_xdmac_tasklet()
1650 			spin_unlock(&atchan->lock);  in at_xdmac_tasklet()
1656 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_tasklet()
1657 		spin_unlock(&atchan->lock);  in at_xdmac_tasklet()
1659 		if (!at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1667 		at_xdmac_advance_work(atchan);  in at_xdmac_tasklet()
1674 	struct at_xdmac_chan	*atchan;  in at_xdmac_interrupt()  local
1696 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1697 			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in at_xdmac_interrupt()
1698 			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);  in at_xdmac_interrupt()
1699 			atchan->irq_status = chan_status & chan_imr;  in at_xdmac_interrupt()
1703 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1706 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_interrupt()
1707 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_interrupt()
1708 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_interrupt()
1709 				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_interrupt()
1710 				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_interrupt()
1711 				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_interrupt()
1713 			if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))  in at_xdmac_interrupt()
1714 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1716 			tasklet_schedule(&atchan->tasklet);  in at_xdmac_interrupt()
1727 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()  local
1729 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1731 	if (!at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_issue_pending()
1732 		at_xdmac_advance_work(atchan);  in at_xdmac_issue_pending()
1740 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()  local
1746 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_config()
1748 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_config()
1755 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()  local
1756 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1761 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))  in at_xdmac_device_pause()
1764 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause()
1765 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1766 	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)  in at_xdmac_device_pause()
1769 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause()
1776 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()  local
1777 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1782 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume()
1783 	if (!at_xdmac_chan_is_paused(atchan)) {  in at_xdmac_device_resume()
1784 		spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1788 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1789 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_resume()
1790 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1798 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()  local
1799 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1804 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1805 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1806 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1810 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)  in at_xdmac_device_terminate_all()
1811 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_device_terminate_all()
1813 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_terminate_all()
1814 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_device_terminate_all()
1815 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1822 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()  local
1827 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1829 	if (at_xdmac_chan_is_enabled(atchan)) {  in at_xdmac_alloc_chan_resources()
1836 	if (!list_empty(&atchan->free_descs_list)) {  in at_xdmac_alloc_chan_resources()
1850 		list_add_tail(&desc->desc_node, &atchan->free_descs_list);  in at_xdmac_alloc_chan_resources()
1858 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1864 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()  local
1868 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {  in at_xdmac_free_chan_resources()
1884 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()  local
1887 		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))  in atmel_xdmac_prepare()
1903 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()  local
1905 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);  in atmel_xdmac_suspend()
1906 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_suspend()
1907 			if (!at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_suspend()
1909 			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in atmel_xdmac_suspend()
1910 			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);  in atmel_xdmac_suspend()
1911 			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);  in atmel_xdmac_suspend()
1924 	struct at_xdmac_chan	*atchan;  in atmel_xdmac_resume()  local
1935 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1936 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in atmel_xdmac_resume()
1942 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1943 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);  in atmel_xdmac_resume()
1944 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_resume()
1945 			if (at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_resume()
1947 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);  in atmel_xdmac_resume()
1948 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);  in atmel_xdmac_resume()
1949 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);  in atmel_xdmac_resume()
1951 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
2067 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()  local
2069 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2070 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
2073 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2074 		atchan->mask = 1 << i;  in at_xdmac_probe()
2076 		spin_lock_init(&atchan->lock);  in at_xdmac_probe()
2077 		INIT_LIST_HEAD(&atchan->xfers_list);  in at_xdmac_probe()
2078 		INIT_LIST_HEAD(&atchan->free_descs_list);  in at_xdmac_probe()
2079 		tasklet_init(&atchan->tasklet, at_xdmac_tasklet,  in at_xdmac_probe()
2080 			     (unsigned long)atchan);  in at_xdmac_probe()
2083 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in at_xdmac_probe()
2128 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()  local
2130 		tasklet_kill(&atchan->tasklet);  in at_xdmac_remove()
2131 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()