380 	void (*start_transfer)(struct xilinx_dma_chan *chan);
381 	int (*stop_transfer)(struct xilinx_dma_chan *chan);
431 	struct xilinx_dma_chan *chan[XILINX_DMA_MAX_CHANS_PER_DEVICE];  member
448 #define to_xilinx_chan(chan) \  argument
449 	container_of(chan, struct xilinx_dma_chan, common)
452 #define xilinx_dma_poll_timeout(chan, reg, val, cond, delay_us, timeout_us) \  argument
453 	readl_poll_timeout(chan->xdev->regs + chan->ctrl_offset + reg, val, \
457 static inline u32 dma_read(struct xilinx_dma_chan *chan, u32 reg)  in dma_read()  argument
459 	return ioread32(chan->xdev->regs + reg);  in dma_read()
462 static inline void dma_write(struct xilinx_dma_chan *chan, u32 reg, u32 value)  in dma_write()  argument
464 	iowrite32(value, chan->xdev->regs + reg);  in dma_write()
467 static inline void vdma_desc_write(struct xilinx_dma_chan *chan, u32 reg,  in vdma_desc_write()  argument
470 	dma_write(chan, chan->desc_offset + reg, value);  in vdma_desc_write()
473 static inline u32 dma_ctrl_read(struct xilinx_dma_chan *chan, u32 reg)  in dma_ctrl_read()  argument
475 	return dma_read(chan, chan->ctrl_offset + reg);  in dma_ctrl_read()
478 static inline void dma_ctrl_write(struct xilinx_dma_chan *chan, u32 reg,  in dma_ctrl_write()  argument
481 	dma_write(chan, chan->ctrl_offset + reg, value);  in dma_ctrl_write()
484 static inline void dma_ctrl_clr(struct xilinx_dma_chan *chan, u32 reg,  in dma_ctrl_clr()  argument
487 	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) & ~clr);  in dma_ctrl_clr()
490 static inline void dma_ctrl_set(struct xilinx_dma_chan *chan, u32 reg,  in dma_ctrl_set()  argument
493 	dma_ctrl_write(chan, reg, dma_ctrl_read(chan, reg) | set);  in dma_ctrl_set()
507 static inline void vdma_desc_write_64(struct xilinx_dma_chan *chan, u32 reg,  in vdma_desc_write_64()  argument
511 	writel(value_lsb, chan->xdev->regs + chan->desc_offset + reg);  in vdma_desc_write_64()
514 	writel(value_msb, chan->xdev->regs + chan->desc_offset + reg + 4);  in vdma_desc_write_64()
517 static inline void dma_writeq(struct xilinx_dma_chan *chan, u32 reg, u64 value)  in dma_writeq()  argument
519 	lo_hi_writeq(value, chan->xdev->regs + chan->ctrl_offset + reg);  in dma_writeq()
522 static inline void xilinx_write(struct xilinx_dma_chan *chan, u32 reg,  in xilinx_write()  argument
525 	if (chan->ext_addr)  in xilinx_write()
526 		dma_writeq(chan, reg, addr);  in xilinx_write()
528 		dma_ctrl_write(chan, reg, addr);  in xilinx_write()
531 static inline void xilinx_axidma_buf(struct xilinx_dma_chan *chan,  in xilinx_axidma_buf()  argument
536 	if (chan->ext_addr) {  in xilinx_axidma_buf()
556 xilinx_vdma_alloc_tx_segment(struct xilinx_dma_chan *chan)  in xilinx_vdma_alloc_tx_segment()  argument
561 	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);  in xilinx_vdma_alloc_tx_segment()
577 xilinx_cdma_alloc_tx_segment(struct xilinx_dma_chan *chan)  in xilinx_cdma_alloc_tx_segment()  argument
582 	segment = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &phys);  in xilinx_cdma_alloc_tx_segment()
598 xilinx_axidma_alloc_tx_segment(struct xilinx_dma_chan *chan)  in xilinx_axidma_alloc_tx_segment()  argument
603 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_axidma_alloc_tx_segment()
604 	if (!list_empty(&chan->free_seg_list)) {  in xilinx_axidma_alloc_tx_segment()
605 		segment = list_first_entry(&chan->free_seg_list,  in xilinx_axidma_alloc_tx_segment()
610 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_axidma_alloc_tx_segment()
631 static void xilinx_dma_free_tx_segment(struct xilinx_dma_chan *chan,  in xilinx_dma_free_tx_segment()  argument
636 	list_add_tail(&segment->node, &chan->free_seg_list);  in xilinx_dma_free_tx_segment()
644 static void xilinx_cdma_free_tx_segment(struct xilinx_dma_chan *chan,  in xilinx_cdma_free_tx_segment()  argument
647 	dma_pool_free(chan->desc_pool, segment, segment->phys);  in xilinx_cdma_free_tx_segment()
655 static void xilinx_vdma_free_tx_segment(struct xilinx_dma_chan *chan,  in xilinx_vdma_free_tx_segment()  argument
658 	dma_pool_free(chan->desc_pool, segment, segment->phys);  in xilinx_vdma_free_tx_segment()
668 xilinx_dma_alloc_tx_descriptor(struct xilinx_dma_chan *chan)  in xilinx_dma_alloc_tx_descriptor()  argument
687 xilinx_dma_free_tx_descriptor(struct xilinx_dma_chan *chan,  in xilinx_dma_free_tx_descriptor()  argument
697 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in xilinx_dma_free_tx_descriptor()
700 			xilinx_vdma_free_tx_segment(chan, segment);  in xilinx_dma_free_tx_descriptor()
702 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_free_tx_descriptor()
706 			xilinx_cdma_free_tx_segment(chan, cdma_segment);  in xilinx_dma_free_tx_descriptor()
712 			xilinx_dma_free_tx_segment(chan, axidma_segment);  in xilinx_dma_free_tx_descriptor()
726 static void xilinx_dma_free_desc_list(struct xilinx_dma_chan *chan,  in xilinx_dma_free_desc_list()  argument
733 		xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_free_desc_list()
741 static void xilinx_dma_free_descriptors(struct xilinx_dma_chan *chan)  in xilinx_dma_free_descriptors()  argument
745 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_free_descriptors()
747 	xilinx_dma_free_desc_list(chan, &chan->pending_list);  in xilinx_dma_free_descriptors()
748 	xilinx_dma_free_desc_list(chan, &chan->done_list);  in xilinx_dma_free_descriptors()
749 	xilinx_dma_free_desc_list(chan, &chan->active_list);  in xilinx_dma_free_descriptors()
751 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_free_descriptors()
760 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_free_chan_resources()  local
763 	dev_dbg(chan->dev, "Free all channel resources.\n");  in xilinx_dma_free_chan_resources()
765 	xilinx_dma_free_descriptors(chan);  in xilinx_dma_free_chan_resources()
767 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_free_chan_resources()
768 		spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_free_chan_resources()
769 		INIT_LIST_HEAD(&chan->free_seg_list);  in xilinx_dma_free_chan_resources()
770 		spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_free_chan_resources()
773 		dma_free_coherent(chan->dev, sizeof(*chan->seg_v) *  in xilinx_dma_free_chan_resources()
774 				  XILINX_DMA_NUM_DESCS, chan->seg_v,  in xilinx_dma_free_chan_resources()
775 				  chan->seg_p);  in xilinx_dma_free_chan_resources()
778 		dma_free_coherent(chan->dev, sizeof(*chan->cyclic_seg_v),  in xilinx_dma_free_chan_resources()
779 				  chan->cyclic_seg_v, chan->cyclic_seg_p);  in xilinx_dma_free_chan_resources()
782 	if (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA) {  in xilinx_dma_free_chan_resources()
783 		dma_pool_destroy(chan->desc_pool);  in xilinx_dma_free_chan_resources()
784 		chan->desc_pool = NULL;  in xilinx_dma_free_chan_resources()
794 static void xilinx_dma_chan_handle_cyclic(struct xilinx_dma_chan *chan,  in xilinx_dma_chan_handle_cyclic()  argument
804 		spin_unlock_irqrestore(&chan->lock, *flags);  in xilinx_dma_chan_handle_cyclic()
806 		spin_lock_irqsave(&chan->lock, *flags);  in xilinx_dma_chan_handle_cyclic()
814 static void xilinx_dma_chan_desc_cleanup(struct xilinx_dma_chan *chan)  in xilinx_dma_chan_desc_cleanup()  argument
819 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_chan_desc_cleanup()
821 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {  in xilinx_dma_chan_desc_cleanup()
825 			xilinx_dma_chan_handle_cyclic(chan, desc, &flags);  in xilinx_dma_chan_desc_cleanup()
835 			spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_chan_desc_cleanup()
837 			spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_chan_desc_cleanup()
842 		xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_chan_desc_cleanup()
845 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_chan_desc_cleanup()
854 	struct xilinx_dma_chan *chan = (struct xilinx_dma_chan *)data;  in xilinx_dma_do_tasklet()  local
856 	xilinx_dma_chan_desc_cleanup(chan);  in xilinx_dma_do_tasklet()
867 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_alloc_chan_resources()  local
871 	if (chan->desc_pool)  in xilinx_dma_alloc_chan_resources()
878 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_alloc_chan_resources()
880 		chan->seg_v = dma_zalloc_coherent(chan->dev,  in xilinx_dma_alloc_chan_resources()
881 						  sizeof(*chan->seg_v) *  in xilinx_dma_alloc_chan_resources()
883 						  &chan->seg_p, GFP_KERNEL);  in xilinx_dma_alloc_chan_resources()
884 		if (!chan->seg_v) {  in xilinx_dma_alloc_chan_resources()
885 			dev_err(chan->dev,  in xilinx_dma_alloc_chan_resources()
887 				chan->id);  in xilinx_dma_alloc_chan_resources()
892 			chan->seg_v[i].hw.next_desc =  in xilinx_dma_alloc_chan_resources()
893 			lower_32_bits(chan->seg_p + sizeof(*chan->seg_v) *  in xilinx_dma_alloc_chan_resources()
895 			chan->seg_v[i].hw.next_desc_msb =  in xilinx_dma_alloc_chan_resources()
896 			upper_32_bits(chan->seg_p + sizeof(*chan->seg_v) *  in xilinx_dma_alloc_chan_resources()
898 			chan->seg_v[i].phys = chan->seg_p +  in xilinx_dma_alloc_chan_resources()
899 				sizeof(*chan->seg_v) * i;  in xilinx_dma_alloc_chan_resources()
900 			list_add_tail(&chan->seg_v[i].node,  in xilinx_dma_alloc_chan_resources()
901 				      &chan->free_seg_list);  in xilinx_dma_alloc_chan_resources()
903 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in xilinx_dma_alloc_chan_resources()
904 		chan->desc_pool = dma_pool_create("xilinx_cdma_desc_pool",  in xilinx_dma_alloc_chan_resources()
905 				   chan->dev,  in xilinx_dma_alloc_chan_resources()
910 		chan->desc_pool = dma_pool_create("xilinx_vdma_desc_pool",  in xilinx_dma_alloc_chan_resources()
911 				     chan->dev,  in xilinx_dma_alloc_chan_resources()
917 	if (!chan->desc_pool &&  in xilinx_dma_alloc_chan_resources()
918 	    (chan->xdev->dma_config->dmatype != XDMA_TYPE_AXIDMA)) {  in xilinx_dma_alloc_chan_resources()
919 		dev_err(chan->dev,  in xilinx_dma_alloc_chan_resources()
921 			chan->id);  in xilinx_dma_alloc_chan_resources()
925 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_alloc_chan_resources()
932 		chan->cyclic_seg_v = dma_zalloc_coherent(chan->dev,  in xilinx_dma_alloc_chan_resources()
933 					sizeof(*chan->cyclic_seg_v),  in xilinx_dma_alloc_chan_resources()
934 					&chan->cyclic_seg_p, GFP_KERNEL);  in xilinx_dma_alloc_chan_resources()
935 		if (!chan->cyclic_seg_v) {  in xilinx_dma_alloc_chan_resources()
936 			dev_err(chan->dev,  in xilinx_dma_alloc_chan_resources()
940 		chan->cyclic_seg_v->phys = chan->cyclic_seg_p;  in xilinx_dma_alloc_chan_resources()
945 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_alloc_chan_resources()
949 		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,  in xilinx_dma_alloc_chan_resources()
953 	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)  in xilinx_dma_alloc_chan_resources()
954 		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,  in xilinx_dma_alloc_chan_resources()
972 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_tx_status()  local
984 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_AXIDMA) {  in xilinx_dma_tx_status()
985 		spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_tx_status()
987 		desc = list_last_entry(&chan->active_list,  in xilinx_dma_tx_status()
989 		if (chan->has_sg) {  in xilinx_dma_tx_status()
996 		spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_tx_status()
998 		chan->residue = residue;  in xilinx_dma_tx_status()
999 		dma_set_residue(txstate, chan->residue);  in xilinx_dma_tx_status()
1011 static int xilinx_dma_stop_transfer(struct xilinx_dma_chan *chan)  in xilinx_dma_stop_transfer()  argument
1015 	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);  in xilinx_dma_stop_transfer()
1018 	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,  in xilinx_dma_stop_transfer()
1029 static int xilinx_cdma_stop_transfer(struct xilinx_dma_chan *chan)  in xilinx_cdma_stop_transfer()  argument
1033 	return xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,  in xilinx_cdma_stop_transfer()
1042 static void xilinx_dma_start(struct xilinx_dma_chan *chan)  in xilinx_dma_start()  argument
1047 	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RUNSTOP);  in xilinx_dma_start()
1050 	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMASR, val,  in xilinx_dma_start()
1055 		dev_err(chan->dev, "Cannot start channel %p: %x\n",  in xilinx_dma_start()
1056 			chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));  in xilinx_dma_start()
1058 		chan->err = true;  in xilinx_dma_start()
1066 static void xilinx_vdma_start_transfer(struct xilinx_dma_chan *chan)  in xilinx_vdma_start_transfer()  argument
1068 	struct xilinx_vdma_config *config = &chan->config;  in xilinx_vdma_start_transfer()
1074 	if (chan->err)  in xilinx_vdma_start_transfer()
1077 	if (!chan->idle)  in xilinx_vdma_start_transfer()
1080 	if (list_empty(&chan->pending_list))  in xilinx_vdma_start_transfer()
1083 	desc = list_first_entry(&chan->pending_list,  in xilinx_vdma_start_transfer()
1085 	tail_desc = list_last_entry(&chan->pending_list,  in xilinx_vdma_start_transfer()
1095 	if (chan->has_sg)  in xilinx_vdma_start_transfer()
1096 		dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,  in xilinx_vdma_start_transfer()
1100 	if (chan->has_vflip) {  in xilinx_vdma_start_transfer()
1101 		reg = dma_read(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP);  in xilinx_vdma_start_transfer()
1104 		dma_write(chan, XILINX_VDMA_REG_ENABLE_VERTICAL_FLIP,  in xilinx_vdma_start_transfer()
1108 	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_vdma_start_transfer()
1119 	if (chan->has_sg || !config->park)  in xilinx_vdma_start_transfer()
1125 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);  in xilinx_vdma_start_transfer()
1127 	j = chan->desc_submitcount;  in xilinx_vdma_start_transfer()
1128 	reg = dma_read(chan, XILINX_DMA_REG_PARK_PTR);  in xilinx_vdma_start_transfer()
1129 	if (chan->direction == DMA_MEM_TO_DEV) {  in xilinx_vdma_start_transfer()
1136 	dma_write(chan, XILINX_DMA_REG_PARK_PTR, reg);  in xilinx_vdma_start_transfer()
1139 	xilinx_dma_start(chan);  in xilinx_vdma_start_transfer()
1141 	if (chan->err)  in xilinx_vdma_start_transfer()
1145 	if (chan->has_sg) {  in xilinx_vdma_start_transfer()
1146 		dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_vdma_start_transfer()
1148 		list_splice_tail_init(&chan->pending_list, &chan->active_list);  in xilinx_vdma_start_transfer()
1149 		chan->desc_pendingcount = 0;  in xilinx_vdma_start_transfer()
1154 		if (chan->desc_submitcount < chan->num_frms)  in xilinx_vdma_start_transfer()
1155 			i = chan->desc_submitcount;  in xilinx_vdma_start_transfer()
1158 			if (chan->ext_addr)  in xilinx_vdma_start_transfer()
1159 				vdma_desc_write_64(chan,  in xilinx_vdma_start_transfer()
1164 				vdma_desc_write(chan,  in xilinx_vdma_start_transfer()
1175 		vdma_desc_write(chan, XILINX_DMA_REG_HSIZE, last->hw.hsize);  in xilinx_vdma_start_transfer()
1176 		vdma_desc_write(chan, XILINX_DMA_REG_FRMDLY_STRIDE,  in xilinx_vdma_start_transfer()
1178 		vdma_desc_write(chan, XILINX_DMA_REG_VSIZE, last->hw.vsize);  in xilinx_vdma_start_transfer()
1180 		chan->desc_submitcount++;  in xilinx_vdma_start_transfer()
1181 		chan->desc_pendingcount--;  in xilinx_vdma_start_transfer()
1183 		list_add_tail(&desc->node, &chan->active_list);  in xilinx_vdma_start_transfer()
1184 		if (chan->desc_submitcount == chan->num_frms)  in xilinx_vdma_start_transfer()
1185 			chan->desc_submitcount = 0;  in xilinx_vdma_start_transfer()
1188 	chan->idle = false;  in xilinx_vdma_start_transfer()
1195 static void xilinx_cdma_start_transfer(struct xilinx_dma_chan *chan)  in xilinx_cdma_start_transfer()  argument
1199 	u32 ctrl_reg = dma_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_cdma_start_transfer()
1201 	if (chan->err)  in xilinx_cdma_start_transfer()
1204 	if (!chan->idle)  in xilinx_cdma_start_transfer()
1207 	if (list_empty(&chan->pending_list))  in xilinx_cdma_start_transfer()
1210 	head_desc = list_first_entry(&chan->pending_list,  in xilinx_cdma_start_transfer()
1212 	tail_desc = list_last_entry(&chan->pending_list,  in xilinx_cdma_start_transfer()
1217 	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {  in xilinx_cdma_start_transfer()
1219 		ctrl_reg |= chan->desc_pendingcount <<  in xilinx_cdma_start_transfer()
1221 		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, ctrl_reg);  in xilinx_cdma_start_transfer()
1224 	if (chan->has_sg) {  in xilinx_cdma_start_transfer()
1225 		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,  in xilinx_cdma_start_transfer()
1228 		dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,  in xilinx_cdma_start_transfer()
1231 		xilinx_write(chan, XILINX_DMA_REG_CURDESC,  in xilinx_cdma_start_transfer()
1235 		xilinx_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_cdma_start_transfer()
1248 		xilinx_write(chan, XILINX_CDMA_REG_SRCADDR, hw->src_addr);  in xilinx_cdma_start_transfer()
1249 		xilinx_write(chan, XILINX_CDMA_REG_DSTADDR, hw->dest_addr);  in xilinx_cdma_start_transfer()
1252 		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,  in xilinx_cdma_start_transfer()
1256 	list_splice_tail_init(&chan->pending_list, &chan->active_list);  in xilinx_cdma_start_transfer()
1257 	chan->desc_pendingcount = 0;  in xilinx_cdma_start_transfer()
1258 	chan->idle = false;  in xilinx_cdma_start_transfer()
1265 static void xilinx_dma_start_transfer(struct xilinx_dma_chan *chan)  in xilinx_dma_start_transfer()  argument
1271 	if (chan->err)  in xilinx_dma_start_transfer()
1274 	if (list_empty(&chan->pending_list))  in xilinx_dma_start_transfer()
1277 	if (!chan->idle)  in xilinx_dma_start_transfer()
1280 	head_desc = list_first_entry(&chan->pending_list,  in xilinx_dma_start_transfer()
1282 	tail_desc = list_last_entry(&chan->pending_list,  in xilinx_dma_start_transfer()
1287 	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_dma_start_transfer()
1289 	if (chan->desc_pendingcount <= XILINX_DMA_COALESCE_MAX) {  in xilinx_dma_start_transfer()
1291 		reg |= chan->desc_pendingcount <<  in xilinx_dma_start_transfer()
1293 		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);  in xilinx_dma_start_transfer()
1296 	if (chan->has_sg && !chan->xdev->mcdma)  in xilinx_dma_start_transfer()
1297 		xilinx_write(chan, XILINX_DMA_REG_CURDESC,  in xilinx_dma_start_transfer()
1300 	if (chan->has_sg && chan->xdev->mcdma) {  in xilinx_dma_start_transfer()
1301 		if (chan->direction == DMA_MEM_TO_DEV) {  in xilinx_dma_start_transfer()
1302 			dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,  in xilinx_dma_start_transfer()
1305 			if (!chan->tdest) {  in xilinx_dma_start_transfer()
1306 				dma_ctrl_write(chan, XILINX_DMA_REG_CURDESC,  in xilinx_dma_start_transfer()
1309 				dma_ctrl_write(chan,  in xilinx_dma_start_transfer()
1310 					XILINX_DMA_MCRX_CDESC(chan->tdest),  in xilinx_dma_start_transfer()
1316 	xilinx_dma_start(chan);  in xilinx_dma_start_transfer()
1318 	if (chan->err)  in xilinx_dma_start_transfer()
1322 	if (chan->has_sg && !chan->xdev->mcdma) {  in xilinx_dma_start_transfer()
1323 		if (chan->cyclic)  in xilinx_dma_start_transfer()
1324 			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_dma_start_transfer()
1325 				     chan->cyclic_seg_v->phys);  in xilinx_dma_start_transfer()
1327 			xilinx_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_dma_start_transfer()
1329 	} else if (chan->has_sg && chan->xdev->mcdma) {  in xilinx_dma_start_transfer()
1330 		if (chan->direction == DMA_MEM_TO_DEV) {  in xilinx_dma_start_transfer()
1331 			dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_dma_start_transfer()
1334 			if (!chan->tdest) {  in xilinx_dma_start_transfer()
1335 				dma_ctrl_write(chan, XILINX_DMA_REG_TAILDESC,  in xilinx_dma_start_transfer()
1338 				dma_ctrl_write(chan,  in xilinx_dma_start_transfer()
1339 					XILINX_DMA_MCRX_TDESC(chan->tdest),  in xilinx_dma_start_transfer()
1352 		xilinx_write(chan, XILINX_DMA_REG_SRCDSTADDR, hw->buf_addr);  in xilinx_dma_start_transfer()
1355 		dma_ctrl_write(chan, XILINX_DMA_REG_BTT,  in xilinx_dma_start_transfer()
1359 	list_splice_tail_init(&chan->pending_list, &chan->active_list);  in xilinx_dma_start_transfer()
1360 	chan->desc_pendingcount = 0;  in xilinx_dma_start_transfer()
1361 	chan->idle = false;  in xilinx_dma_start_transfer()
1370 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_issue_pending()  local
1373 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_issue_pending()
1374 	chan->start_transfer(chan);  in xilinx_dma_issue_pending()
1375 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_issue_pending()
1384 static void xilinx_dma_complete_descriptor(struct xilinx_dma_chan *chan)  in xilinx_dma_complete_descriptor()  argument
1389 	if (list_empty(&chan->active_list))  in xilinx_dma_complete_descriptor()
1392 	list_for_each_entry_safe(desc, next, &chan->active_list, node) {  in xilinx_dma_complete_descriptor()
1396 		list_add_tail(&desc->node, &chan->done_list);  in xilinx_dma_complete_descriptor()
1406 static int xilinx_dma_reset(struct xilinx_dma_chan *chan)  in xilinx_dma_reset()  argument
1411 	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR, XILINX_DMA_DMACR_RESET);  in xilinx_dma_reset()
1414 	err = xilinx_dma_poll_timeout(chan, XILINX_DMA_REG_DMACR, tmp,  in xilinx_dma_reset()
1419 		dev_err(chan->dev, "reset timeout, cr %x, sr %x\n",  in xilinx_dma_reset()
1420 			dma_ctrl_read(chan, XILINX_DMA_REG_DMACR),  in xilinx_dma_reset()
1421 			dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));  in xilinx_dma_reset()
1425 	chan->err = false;  in xilinx_dma_reset()
1426 	chan->idle = true;  in xilinx_dma_reset()
1427 	chan->desc_submitcount = 0;  in xilinx_dma_reset()
1438 static int xilinx_dma_chan_reset(struct xilinx_dma_chan *chan)  in xilinx_dma_chan_reset()  argument
1443 	err = xilinx_dma_reset(chan);  in xilinx_dma_chan_reset()
1448 	dma_ctrl_set(chan, XILINX_DMA_REG_DMACR,  in xilinx_dma_chan_reset()
1463 	struct xilinx_dma_chan *chan = data;  in xilinx_dma_irq_handler()  local
1467 	status = dma_ctrl_read(chan, XILINX_DMA_REG_DMASR);  in xilinx_dma_irq_handler()
1471 	dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,  in xilinx_dma_irq_handler()
1484 		dma_ctrl_write(chan, XILINX_DMA_REG_DMASR,  in xilinx_dma_irq_handler()
1487 		if (!chan->flush_on_fsync ||  in xilinx_dma_irq_handler()
1489 			dev_err(chan->dev,  in xilinx_dma_irq_handler()
1491 				chan, errors,  in xilinx_dma_irq_handler()
1492 				dma_ctrl_read(chan, XILINX_DMA_REG_CURDESC),  in xilinx_dma_irq_handler()
1493 				dma_ctrl_read(chan, XILINX_DMA_REG_TAILDESC));  in xilinx_dma_irq_handler()
1494 			chan->err = true;  in xilinx_dma_irq_handler()
1503 		dev_dbg(chan->dev, "Inter-packet latency too long\n");  in xilinx_dma_irq_handler()
1507 		spin_lock(&chan->lock);  in xilinx_dma_irq_handler()
1508 		xilinx_dma_complete_descriptor(chan);  in xilinx_dma_irq_handler()
1509 		chan->idle = true;  in xilinx_dma_irq_handler()
1510 		chan->start_transfer(chan);  in xilinx_dma_irq_handler()
1511 		spin_unlock(&chan->lock);  in xilinx_dma_irq_handler()
1514 	tasklet_schedule(&chan->tasklet);  in xilinx_dma_irq_handler()
1523 static void append_desc_queue(struct xilinx_dma_chan *chan,  in append_desc_queue()  argument
1531 	if (list_empty(&chan->pending_list))  in append_desc_queue()
1538 	tail_desc = list_last_entry(&chan->pending_list,  in append_desc_queue()
1540 	if (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA) {  in append_desc_queue()
1545 	} else if (chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) {  in append_desc_queue()
1562 	list_add_tail(&desc->node, &chan->pending_list);  in append_desc_queue()
1563 	chan->desc_pendingcount++;  in append_desc_queue()
1565 	if (chan->has_sg && (chan->xdev->dma_config->dmatype == XDMA_TYPE_VDMA)  in append_desc_queue()
1566 	    && unlikely(chan->desc_pendingcount > chan->num_frms)) {  in append_desc_queue()
1567 		dev_dbg(chan->dev, "desc pendingcount is too high\n");  in append_desc_queue()
1568 		chan->desc_pendingcount = chan->num_frms;  in append_desc_queue()
1581 	struct xilinx_dma_chan *chan = to_xilinx_chan(tx->chan);  in xilinx_dma_tx_submit()  local
1586 	if (chan->cyclic) {  in xilinx_dma_tx_submit()
1587 		xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_tx_submit()
1591 	if (chan->err) {  in xilinx_dma_tx_submit()
1596 		err = xilinx_dma_chan_reset(chan);  in xilinx_dma_tx_submit()
1601 	spin_lock_irqsave(&chan->lock, flags);  in xilinx_dma_tx_submit()
1606 	append_desc_queue(chan, desc);  in xilinx_dma_tx_submit()
1609 		chan->cyclic = true;  in xilinx_dma_tx_submit()
1611 	spin_unlock_irqrestore(&chan->lock, flags);  in xilinx_dma_tx_submit()
1630 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_dma_prep_interleaved()  local
1645 	desc = xilinx_dma_alloc_tx_descriptor(chan);  in xilinx_vdma_dma_prep_interleaved()
1649 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_vdma_dma_prep_interleaved()
1654 	segment = xilinx_vdma_alloc_tx_segment(chan);  in xilinx_vdma_dma_prep_interleaved()
1664 	hw->stride |= chan->config.frm_dly <<  in xilinx_vdma_dma_prep_interleaved()
1668 		if (chan->ext_addr) {  in xilinx_vdma_dma_prep_interleaved()
1675 		if (chan->ext_addr) {  in xilinx_vdma_dma_prep_interleaved()
1694 	xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_vdma_dma_prep_interleaved()
1712 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_cdma_prep_memcpy()  local
1720 	desc = xilinx_dma_alloc_tx_descriptor(chan);  in xilinx_cdma_prep_memcpy()
1724 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_cdma_prep_memcpy()
1728 	segment = xilinx_cdma_alloc_tx_segment(chan);  in xilinx_cdma_prep_memcpy()
1736 	if (chan->ext_addr) {  in xilinx_cdma_prep_memcpy()
1750 	xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_cdma_prep_memcpy()
1770 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_prep_slave_sg()  local
1783 	desc = xilinx_dma_alloc_tx_descriptor(chan);  in xilinx_dma_prep_slave_sg()
1787 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_dma_prep_slave_sg()
1799 			segment = xilinx_axidma_alloc_tx_segment(chan);  in xilinx_dma_prep_slave_sg()
1812 			xilinx_axidma_buf(chan, hw, sg_dma_address(sg),  in xilinx_dma_prep_slave_sg()
1817 			if (chan->direction == DMA_MEM_TO_DEV) {  in xilinx_dma_prep_slave_sg()
1838 	if (chan->direction == DMA_MEM_TO_DEV) {  in xilinx_dma_prep_slave_sg()
1849 	xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_prep_slave_sg()
1869 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_prep_dma_cyclic()  local
1889 	desc = xilinx_dma_alloc_tx_descriptor(chan);  in xilinx_dma_prep_dma_cyclic()
1893 	chan->direction = direction;  in xilinx_dma_prep_dma_cyclic()
1894 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_dma_prep_dma_cyclic()
1904 			segment = xilinx_axidma_alloc_tx_segment(chan);  in xilinx_dma_prep_dma_cyclic()
1915 			xilinx_axidma_buf(chan, hw, buf_addr, sg_used,  in xilinx_dma_prep_dma_cyclic()
1938 	reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_dma_prep_dma_cyclic()
1940 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);  in xilinx_dma_prep_dma_cyclic()
1956 	xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_prep_dma_cyclic()
1974 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_prep_interleaved()  local
1989 	desc = xilinx_dma_alloc_tx_descriptor(chan);  in xilinx_dma_prep_interleaved()
1993 	chan->direction = xt->dir;  in xilinx_dma_prep_interleaved()
1994 	dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in xilinx_dma_prep_interleaved()
1998 	segment = xilinx_axidma_alloc_tx_segment(chan);  in xilinx_dma_prep_interleaved()
2010 	hw->mcdma_control = chan->tdest & XILINX_DMA_BD_TDEST_MASK;  in xilinx_dma_prep_interleaved()
2040 	xilinx_dma_free_tx_descriptor(chan, desc);  in xilinx_dma_prep_interleaved()
2052 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_dma_terminate_all()  local
2056 	if (chan->cyclic)  in xilinx_dma_terminate_all()
2057 		xilinx_dma_chan_reset(chan);  in xilinx_dma_terminate_all()
2059 	err = chan->stop_transfer(chan);  in xilinx_dma_terminate_all()
2061 		dev_err(chan->dev, "Cannot stop channel %p: %x\n",  in xilinx_dma_terminate_all()
2062 			chan, dma_ctrl_read(chan, XILINX_DMA_REG_DMASR));  in xilinx_dma_terminate_all()
2063 		chan->err = true;  in xilinx_dma_terminate_all()
2067 	xilinx_dma_free_descriptors(chan);  in xilinx_dma_terminate_all()
2068 	chan->idle = true;  in xilinx_dma_terminate_all()
2070 	if (chan->cyclic) {  in xilinx_dma_terminate_all()
2071 		reg = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_dma_terminate_all()
2073 		dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, reg);  in xilinx_dma_terminate_all()
2074 		chan->cyclic = false;  in xilinx_dma_terminate_all()
2077 	if ((chan->xdev->dma_config->dmatype == XDMA_TYPE_CDMA) && chan->has_sg)  in xilinx_dma_terminate_all()
2078 		dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,  in xilinx_dma_terminate_all()
2100 	struct xilinx_dma_chan *chan = to_xilinx_chan(dchan);  in xilinx_vdma_channel_set_config()  local
2104 		return xilinx_dma_chan_reset(chan);  in xilinx_vdma_channel_set_config()
2106 	dmacr = dma_ctrl_read(chan, XILINX_DMA_REG_DMACR);  in xilinx_vdma_channel_set_config()
2108 	chan->config.frm_dly = cfg->frm_dly;  in xilinx_vdma_channel_set_config()
2109 	chan->config.park = cfg->park;  in xilinx_vdma_channel_set_config()
2112 	chan->config.gen_lock = cfg->gen_lock;  in xilinx_vdma_channel_set_config()
2113 	chan->config.master = cfg->master;  in xilinx_vdma_channel_set_config()
2115 	if (cfg->gen_lock && chan->genlock) {  in xilinx_vdma_channel_set_config()
2120 	chan->config.frm_cnt_en = cfg->frm_cnt_en;  in xilinx_vdma_channel_set_config()
2121 	chan->config.vflip_en = cfg->vflip_en;  in xilinx_vdma_channel_set_config()
2124 		chan->config.park_frm = cfg->park_frm;  in xilinx_vdma_channel_set_config()
2126 		chan->config.park_frm = -1;  in xilinx_vdma_channel_set_config()
2128 	chan->config.coalesc = cfg->coalesc;  in xilinx_vdma_channel_set_config()
2129 	chan->config.delay = cfg->delay;  in xilinx_vdma_channel_set_config()
2133 		chan->config.coalesc = cfg->coalesc;  in xilinx_vdma_channel_set_config()
2138 		chan->config.delay = cfg->delay;  in xilinx_vdma_channel_set_config()
2145 	dma_ctrl_write(chan, XILINX_DMA_REG_DMACR, dmacr);  in xilinx_vdma_channel_set_config()
2159 static void xilinx_dma_chan_remove(struct xilinx_dma_chan *chan)  in xilinx_dma_chan_remove()  argument
2162 	dma_ctrl_clr(chan, XILINX_DMA_REG_DMACR,  in xilinx_dma_chan_remove()
2165 	if (chan->irq > 0)  in xilinx_dma_chan_remove()
2166 		free_irq(chan->irq, chan);  in xilinx_dma_chan_remove()
2168 	tasklet_kill(&chan->tasklet);  in xilinx_dma_chan_remove()
2170 	list_del(&chan->common.device_node);  in xilinx_dma_chan_remove()
2376 	struct xilinx_dma_chan *chan;  in xilinx_dma_chan_probe()  local
2382 	chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL);  in xilinx_dma_chan_probe()
2383 	if (!chan)  in xilinx_dma_chan_probe()
2386 	chan->dev = xdev->dev;  in xilinx_dma_chan_probe()
2387 	chan->xdev = xdev;  in xilinx_dma_chan_probe()
2388 	chan->has_sg = xdev->has_sg;  in xilinx_dma_chan_probe()
2389 	chan->desc_pendingcount = 0x0;  in xilinx_dma_chan_probe()
2390 	chan->ext_addr = xdev->ext_addr;  in xilinx_dma_chan_probe()
2396 	chan->idle = true;  in xilinx_dma_chan_probe()
2398 	spin_lock_init(&chan->lock);  in xilinx_dma_chan_probe()
2399 	INIT_LIST_HEAD(&chan->pending_list);  in xilinx_dma_chan_probe()
2400 	INIT_LIST_HEAD(&chan->done_list);  in xilinx_dma_chan_probe()
2401 	INIT_LIST_HEAD(&chan->active_list);  in xilinx_dma_chan_probe()
2402 	INIT_LIST_HEAD(&chan->free_seg_list);  in xilinx_dma_chan_probe()
2407 	chan->genlock = of_property_read_bool(node, "xlnx,genlock-mode");  in xilinx_dma_chan_probe()
2426 		chan->direction = DMA_MEM_TO_DEV;  in xilinx_dma_chan_probe()
2427 		chan->id = chan_id;  in xilinx_dma_chan_probe()
2428 		chan->tdest = chan_id;  in xilinx_dma_chan_probe()
2430 		chan->ctrl_offset = XILINX_DMA_MM2S_CTRL_OFFSET;  in xilinx_dma_chan_probe()
2432 			chan->desc_offset = XILINX_VDMA_MM2S_DESC_OFFSET;  in xilinx_dma_chan_probe()
2433 			chan->config.park = 1;  in xilinx_dma_chan_probe()
2437 				chan->flush_on_fsync = true;  in xilinx_dma_chan_probe()
2443 		chan->direction = DMA_DEV_TO_MEM;  in xilinx_dma_chan_probe()
2444 		chan->id = chan_id;  in xilinx_dma_chan_probe()
2445 		chan->tdest = chan_id - xdev->nr_channels;  in xilinx_dma_chan_probe()
2446 		chan->has_vflip = of_property_read_bool(node,  in xilinx_dma_chan_probe()
2448 		if (chan->has_vflip) {  in xilinx_dma_chan_probe()
2449 			chan->config.vflip_en = dma_read(chan,  in xilinx_dma_chan_probe()
2454 		chan->ctrl_offset = XILINX_DMA_S2MM_CTRL_OFFSET;  in xilinx_dma_chan_probe()
2456 			chan->desc_offset = XILINX_VDMA_S2MM_DESC_OFFSET;  in xilinx_dma_chan_probe()
2457 			chan->config.park = 1;  in xilinx_dma_chan_probe()
2461 				chan->flush_on_fsync = true;  in xilinx_dma_chan_probe()
2469 	chan->irq = irq_of_parse_and_map(node, 0);  in xilinx_dma_chan_probe()
2470 	err = request_irq(chan->irq, xilinx_dma_irq_handler, IRQF_SHARED,  in xilinx_dma_chan_probe()
2471 			  "xilinx-dma-controller", chan);  in xilinx_dma_chan_probe()
2473 		dev_err(xdev->dev, "unable to request IRQ %d\n", chan->irq);  in xilinx_dma_chan_probe()
2478 		chan->start_transfer = xilinx_dma_start_transfer;  in xilinx_dma_chan_probe()
2479 		chan->stop_transfer = xilinx_dma_stop_transfer;  in xilinx_dma_chan_probe()
2481 		chan->start_transfer = xilinx_cdma_start_transfer;  in xilinx_dma_chan_probe()
2482 		chan->stop_transfer = xilinx_cdma_stop_transfer;  in xilinx_dma_chan_probe()
2484 		chan->start_transfer = xilinx_vdma_start_transfer;  in xilinx_dma_chan_probe()
2485 		chan->stop_transfer = xilinx_dma_stop_transfer;  in xilinx_dma_chan_probe()
2489 	tasklet_init(&chan->tasklet, xilinx_dma_do_tasklet,  in xilinx_dma_chan_probe()
2490 			(unsigned long)chan);  in xilinx_dma_chan_probe()
2496 	chan->common.device = &xdev->common;  in xilinx_dma_chan_probe()
2498 	list_add_tail(&chan->common.device_node, &xdev->common.channels);  in xilinx_dma_chan_probe()
2499 	xdev->chan[chan->id] = chan;  in xilinx_dma_chan_probe()
2502 	err = xilinx_dma_chan_reset(chan);  in xilinx_dma_chan_probe()
2551 	if (chan_id >= xdev->nr_channels || !xdev->chan[chan_id])  in of_dma_xilinx_xlate()
2554 	return dma_get_slave_channel(&xdev->chan[chan_id]->common);  in of_dma_xilinx_xlate()
2703 			if (xdev->chan[i])  in xilinx_dma_probe()
2704 				xdev->chan[i]->num_frms = num_frames;  in xilinx_dma_probe()
2731 		if (xdev->chan[i])  in xilinx_dma_probe()
2732 			xilinx_dma_chan_remove(xdev->chan[i]);  in xilinx_dma_probe()
2753 		if (xdev->chan[i])  in xilinx_dma_remove()
2754 			xilinx_dma_chan_remove(xdev->chan[i]);  in xilinx_dma_remove()