266 #define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))  argument
267 #define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))  argument
289 static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_cyclic()  argument
291 	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_chan_is_cyclic()
294 static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_paused()  argument
296 	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_chan_is_paused()
321 static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)  in at_xdmac_chan_is_enabled()  argument
323 	return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;  in at_xdmac_chan_is_enabled()
338 static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_start_xfer()  argument
341 	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_start_xfer()
344 	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);  in at_xdmac_start_xfer()
346 	if (at_xdmac_chan_is_enabled(atchan))  in at_xdmac_start_xfer()
354 	      | AT_XDMAC_CNDA_NDAIF(atchan->memif);  in at_xdmac_start_xfer()
355 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);  in at_xdmac_start_xfer()
362 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
374 	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);  in at_xdmac_start_xfer()
379 	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);  in at_xdmac_start_xfer()
381 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
383 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
384 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
385 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
386 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
387 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
388 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
390 	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);  in at_xdmac_start_xfer()
396 	if (at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_start_xfer()
397 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
400 		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,  in at_xdmac_start_xfer()
402 	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);  in at_xdmac_start_xfer()
403 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
404 		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);  in at_xdmac_start_xfer()
406 	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in at_xdmac_start_xfer()
408 	dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_start_xfer()
410 		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_start_xfer()
411 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_start_xfer()
412 		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_start_xfer()
413 		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_start_xfer()
414 		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_start_xfer()
415 		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_start_xfer()
422 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);  in at_xdmac_tx_submit()  local
426 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
430 		 __func__, atchan, desc);  in at_xdmac_tx_submit()
431 	list_add_tail(&desc->xfer_node, &atchan->xfers_list);  in at_xdmac_tx_submit()
432 	if (list_is_singular(&atchan->xfers_list))  in at_xdmac_tx_submit()
433 		at_xdmac_start_xfer(atchan, desc);  in at_xdmac_tx_submit()
435 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_tx_submit()
467 static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)  in at_xdmac_get_desc()  argument
471 	if (list_empty(&atchan->free_descs_list)) {  in at_xdmac_get_desc()
472 		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);  in at_xdmac_get_desc()
474 		desc = list_first_entry(&atchan->free_descs_list,  in at_xdmac_get_desc()
514 	struct at_xdmac_chan	*atchan;  in at_xdmac_xlate()  local
529 	atchan = to_at_xdmac_chan(chan);  in at_xdmac_xlate()
530 	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);  in at_xdmac_xlate()
531 	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);  in at_xdmac_xlate()
532 	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);  in at_xdmac_xlate()
534 		 atchan->memif, atchan->perif, atchan->perid);  in at_xdmac_xlate()
542 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_compute_chan_conf()  local
546 		atchan->cfg =  in at_xdmac_compute_chan_conf()
547 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
550 			| AT_XDMAC_CC_DIF(atchan->memif)  in at_xdmac_compute_chan_conf()
551 			| AT_XDMAC_CC_SIF(atchan->perif)  in at_xdmac_compute_chan_conf()
556 		csize = ffs(atchan->sconfig.src_maxburst) - 1;  in at_xdmac_compute_chan_conf()
561 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
562 		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;  in at_xdmac_compute_chan_conf()
567 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
569 		atchan->cfg =  in at_xdmac_compute_chan_conf()
570 			AT91_XDMAC_DT_PERID(atchan->perid)  in at_xdmac_compute_chan_conf()
573 			| AT_XDMAC_CC_DIF(atchan->perif)  in at_xdmac_compute_chan_conf()
574 			| AT_XDMAC_CC_SIF(atchan->memif)  in at_xdmac_compute_chan_conf()
579 		csize = ffs(atchan->sconfig.dst_maxburst) - 1;  in at_xdmac_compute_chan_conf()
584 		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);  in at_xdmac_compute_chan_conf()
585 		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;  in at_xdmac_compute_chan_conf()
590 		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);  in at_xdmac_compute_chan_conf()
593 	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);  in at_xdmac_compute_chan_conf()
619 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_set_slave_config()  local
626 	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));  in at_xdmac_set_slave_config()
636 	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_slave_sg()  local
658 	spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
677 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_slave_sg()
681 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_slave_sg()
687 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_slave_sg()
691 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_slave_sg()
693 		dwidth = at_xdmac_get_dwidth(atchan->cfg);  in at_xdmac_prep_slave_sg()
701 		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |  in at_xdmac_prep_slave_sg()
728 	spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_slave_sg()
738 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_cyclic()  local
753 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {  in at_xdmac_prep_dma_cyclic()
764 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
765 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_cyclic()
769 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_cyclic()
770 			spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
773 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_cyclic()
779 			desc->lld.mbr_sa = atchan->sconfig.src_addr;  in at_xdmac_prep_dma_cyclic()
783 			desc->lld.mbr_da = atchan->sconfig.dst_addr;  in at_xdmac_prep_dma_cyclic()
785 		desc->lld.mbr_cfg = atchan->cfg;  in at_xdmac_prep_dma_cyclic()
847 				struct at_xdmac_chan *atchan,  in at_xdmac_interleaved_queue_desc()  argument
902 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
903 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_interleaved_queue_desc()
904 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_interleaved_queue_desc()
942 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_interleaved()  local
967 		first = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
996 			desc = at_xdmac_interleaved_queue_desc(chan, atchan,  in at_xdmac_prep_interleaved()
1002 						 &atchan->free_descs_list);  in at_xdmac_prep_interleaved()
1035 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memcpy()  local
1074 		spin_lock_irqsave(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1075 		desc = at_xdmac_get_desc(atchan);  in at_xdmac_prep_dma_memcpy()
1076 		spin_unlock_irqrestore(&atchan->lock, irqflags);  in at_xdmac_prep_dma_memcpy()
1080 				list_splice_init(&first->descs_list, &atchan->free_descs_list);  in at_xdmac_prep_dma_memcpy()
1136 							 struct at_xdmac_chan *atchan,  in at_xdmac_memset_create_desc()  argument
1175 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1176 	desc = at_xdmac_get_desc(atchan);  in at_xdmac_memset_create_desc()
1177 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_memset_create_desc()
1207 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset()  local
1216 	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);  in at_xdmac_prep_dma_memset()
1231 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_prep_dma_memset_sg()  local
1249 		desc = at_xdmac_memset_create_desc(chan, atchan,  in at_xdmac_prep_dma_memset_sg()
1255 					 &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1304 					      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1367 				      &atchan->free_descs_list);  in at_xdmac_prep_dma_memset_sg()
1392 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_tx_status()  local
1393 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_tx_status()
1410 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_tx_status()
1412 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_tx_status()
1440 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1441 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1472 		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1474 		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);  in at_xdmac_tx_status()
1476 		initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);  in at_xdmac_tx_status()
1478 		cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;  in at_xdmac_tx_status()
1498 		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);  in at_xdmac_tx_status()
1499 		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))  in at_xdmac_tx_status()
1524 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_tx_status()
1529 static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,  in at_xdmac_remove_xfer()  argument
1532 	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_remove_xfer()
1539 	list_splice_init(&desc->descs_list, &atchan->free_descs_list);  in at_xdmac_remove_xfer()
1542 static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)  in at_xdmac_advance_work()  argument
1547 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_advance_work()
1553 	if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {  in at_xdmac_advance_work()
1554 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_advance_work()
1557 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_advance_work()
1559 			at_xdmac_start_xfer(atchan, desc);  in at_xdmac_advance_work()
1562 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_advance_work()
1565 static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)  in at_xdmac_handle_cyclic()  argument
1570 	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);  in at_xdmac_handle_cyclic()
1579 	struct at_xdmac_chan	*atchan = (struct at_xdmac_chan *)data;  in at_xdmac_tasklet()  local
1583 	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08lx\n",  in at_xdmac_tasklet()
1584 		 __func__, atchan->status);  in at_xdmac_tasklet()
1590 	if (at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1591 		at_xdmac_handle_cyclic(atchan);  in at_xdmac_tasklet()
1592 	} else if ((atchan->status & AT_XDMAC_CIS_LIS)  in at_xdmac_tasklet()
1593 		   || (atchan->status & error_mask)) {  in at_xdmac_tasklet()
1596 		if (atchan->status & AT_XDMAC_CIS_RBEIS)  in at_xdmac_tasklet()
1597 			dev_err(chan2dev(&atchan->chan), "read bus error!!!");  in at_xdmac_tasklet()
1598 		if (atchan->status & AT_XDMAC_CIS_WBEIS)  in at_xdmac_tasklet()
1599 			dev_err(chan2dev(&atchan->chan), "write bus error!!!");  in at_xdmac_tasklet()
1600 		if (atchan->status & AT_XDMAC_CIS_ROIS)  in at_xdmac_tasklet()
1601 			dev_err(chan2dev(&atchan->chan), "request overflow error!!!");  in at_xdmac_tasklet()
1603 		spin_lock_bh(&atchan->lock);  in at_xdmac_tasklet()
1604 		desc = list_first_entry(&atchan->xfers_list,  in at_xdmac_tasklet()
1607 		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);  in at_xdmac_tasklet()
1612 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_tasklet()
1613 		spin_unlock_bh(&atchan->lock);  in at_xdmac_tasklet()
1615 		if (!at_xdmac_chan_is_cyclic(atchan)) {  in at_xdmac_tasklet()
1623 		at_xdmac_advance_work(atchan);  in at_xdmac_tasklet()
1630 	struct at_xdmac_chan	*atchan;  in at_xdmac_interrupt()  local
1652 			atchan = &atxdmac->chan[i];  in at_xdmac_interrupt()
1653 			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in at_xdmac_interrupt()
1654 			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);  in at_xdmac_interrupt()
1655 			atchan->status = chan_status & chan_imr;  in at_xdmac_interrupt()
1659 			dev_vdbg(chan2dev(&atchan->chan),  in at_xdmac_interrupt()
1662 				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),  in at_xdmac_interrupt()
1663 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),  in at_xdmac_interrupt()
1664 				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),  in at_xdmac_interrupt()
1665 				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),  in at_xdmac_interrupt()
1666 				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),  in at_xdmac_interrupt()
1667 				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));  in at_xdmac_interrupt()
1669 			if (atchan->status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))  in at_xdmac_interrupt()
1670 				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_interrupt()
1672 			tasklet_schedule(&atchan->tasklet);  in at_xdmac_interrupt()
1683 	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);  in at_xdmac_issue_pending()  local
1685 	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);  in at_xdmac_issue_pending()
1687 	if (!at_xdmac_chan_is_cyclic(atchan))  in at_xdmac_issue_pending()
1688 		at_xdmac_advance_work(atchan);  in at_xdmac_issue_pending()
1696 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_config()  local
1702 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_config()
1704 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_config()
1711 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_pause()  local
1712 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_pause()
1717 	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))  in at_xdmac_device_pause()
1720 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_pause()
1721 	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);  in at_xdmac_device_pause()
1722 	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)  in at_xdmac_device_pause()
1725 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_pause()
1732 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_resume()  local
1733 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_resume()
1738 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_resume()
1739 	if (!at_xdmac_chan_is_paused(atchan)) {  in at_xdmac_device_resume()
1740 		spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1744 	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);  in at_xdmac_device_resume()
1745 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_resume()
1746 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_resume()
1754 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_device_terminate_all()  local
1755 	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);  in at_xdmac_device_terminate_all()
1760 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1761 	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);  in at_xdmac_device_terminate_all()
1762 	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)  in at_xdmac_device_terminate_all()
1766 	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)  in at_xdmac_device_terminate_all()
1767 		at_xdmac_remove_xfer(atchan, desc);  in at_xdmac_device_terminate_all()
1769 	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);  in at_xdmac_device_terminate_all()
1770 	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);  in at_xdmac_device_terminate_all()
1771 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_device_terminate_all()
1778 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_alloc_chan_resources()  local
1783 	spin_lock_irqsave(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1785 	if (at_xdmac_chan_is_enabled(atchan)) {  in at_xdmac_alloc_chan_resources()
1792 	if (!list_empty(&atchan->free_descs_list)) {  in at_xdmac_alloc_chan_resources()
1806 		list_add_tail(&desc->desc_node, &atchan->free_descs_list);  in at_xdmac_alloc_chan_resources()
1814 	spin_unlock_irqrestore(&atchan->lock, flags);  in at_xdmac_alloc_chan_resources()
1820 	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in at_xdmac_free_chan_resources()  local
1824 	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {  in at_xdmac_free_chan_resources()
1840 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_prepare()  local
1843 		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))  in atmel_xdmac_prepare()
1859 		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_suspend()  local
1861 		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);  in atmel_xdmac_suspend()
1862 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_suspend()
1863 			if (!at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_suspend()
1865 			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);  in atmel_xdmac_suspend()
1866 			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);  in atmel_xdmac_suspend()
1867 			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);  in atmel_xdmac_suspend()
1880 	struct at_xdmac_chan	*atchan;  in atmel_xdmac_resume()  local
1891 		atchan = &atxdmac->chan[i];  in atmel_xdmac_resume()
1892 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in atmel_xdmac_resume()
1898 		atchan = to_at_xdmac_chan(chan);  in atmel_xdmac_resume()
1899 		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);  in atmel_xdmac_resume()
1900 		if (at_xdmac_chan_is_cyclic(atchan)) {  in atmel_xdmac_resume()
1901 			if (at_xdmac_chan_is_paused(atchan))  in atmel_xdmac_resume()
1903 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);  in atmel_xdmac_resume()
1904 			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);  in atmel_xdmac_resume()
1905 			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);  in atmel_xdmac_resume()
1907 			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);  in atmel_xdmac_resume()
2023 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_probe()  local
2025 		atchan->chan.device = &atxdmac->dma;  in at_xdmac_probe()
2026 		list_add_tail(&atchan->chan.device_node,  in at_xdmac_probe()
2029 		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);  in at_xdmac_probe()
2030 		atchan->mask = 1 << i;  in at_xdmac_probe()
2032 		spin_lock_init(&atchan->lock);  in at_xdmac_probe()
2033 		INIT_LIST_HEAD(&atchan->xfers_list);  in at_xdmac_probe()
2034 		INIT_LIST_HEAD(&atchan->free_descs_list);  in at_xdmac_probe()
2035 		tasklet_init(&atchan->tasklet, at_xdmac_tasklet,  in at_xdmac_probe()
2036 			     (unsigned long)atchan);  in at_xdmac_probe()
2039 		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))  in at_xdmac_probe()
2084 		struct at_xdmac_chan *atchan = &atxdmac->chan[i];  in at_xdmac_remove()  local
2086 		tasklet_kill(&atchan->tasklet);  in at_xdmac_remove()
2087 		at_xdmac_free_chan_resources(&atchan->chan);  in at_xdmac_remove()