227 	struct edma_desc		*edesc;  member
766 	struct edma_desc *edesc;  in edma_execute()  local
770 	if (!echan->edesc) {  in edma_execute()
776 		echan->edesc = to_edma_desc(&vdesc->tx);  in edma_execute()
779 	edesc = echan->edesc;  in edma_execute()
782 	left = edesc->pset_nr - edesc->processed;  in edma_execute()
784 	edesc->sg_len = 0;  in edma_execute()
788 		j = i + edesc->processed;  in edma_execute()
789 		edma_write_slot(ecc, echan->slot[i], &edesc->pset[j].param);  in edma_execute()
790 		edesc->sg_len += edesc->pset[j].len;  in edma_execute()
804 			 edesc->pset[j].param.opt,  in edma_execute()
805 			 edesc->pset[j].param.src,  in edma_execute()
806 			 edesc->pset[j].param.dst,  in edma_execute()
807 			 edesc->pset[j].param.a_b_cnt,  in edma_execute()
808 			 edesc->pset[j].param.ccnt,  in edma_execute()
809 			 edesc->pset[j].param.src_dst_bidx,  in edma_execute()
810 			 edesc->pset[j].param.src_dst_cidx,  in edma_execute()
811 			 edesc->pset[j].param.link_bcntrld);  in edma_execute()
817 	edesc->processed += nslots;  in edma_execute()
824 	if (edesc->processed == edesc->pset_nr) {  in edma_execute()
825 		if (edesc->cyclic)  in edma_execute()
844 	} else if (edesc->processed <= MAX_NR_SG) {  in edma_execute()
850 			echan->ch_num, edesc->processed);  in edma_execute()
868 	if (echan->edesc) {  in edma_terminate_all()
871 		if (!echan->tc && echan->edesc->cyclic)  in edma_terminate_all()
874 		vchan_terminate_vdesc(&echan->edesc->vdesc);  in edma_terminate_all()
875 		echan->edesc = NULL;  in edma_terminate_all()
914 	if (!echan->edesc)  in edma_dma_pause()
1060 	struct edma_desc *edesc;  in edma_prep_slave_sg()  local
1088 	edesc = kzalloc(struct_size(edesc, pset, sg_len), GFP_ATOMIC);  in edma_prep_slave_sg()
1089 	if (!edesc)  in edma_prep_slave_sg()
1092 	edesc->pset_nr = sg_len;  in edma_prep_slave_sg()
1093 	edesc->residue = 0;  in edma_prep_slave_sg()
1094 	edesc->direction = direction;  in edma_prep_slave_sg()
1095 	edesc->echan = echan;  in edma_prep_slave_sg()
1105 				kfree(edesc);  in edma_prep_slave_sg()
1121 		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,  in edma_prep_slave_sg()
1125 			kfree(edesc);  in edma_prep_slave_sg()
1129 		edesc->absync = ret;  in edma_prep_slave_sg()
1130 		edesc->residue += sg_dma_len(sg);  in edma_prep_slave_sg()
1134 			edesc->pset[i].param.opt |= TCINTEN;  in edma_prep_slave_sg()
1142 			edesc->pset[i].param.opt |= (TCINTEN | TCCMODE);  in edma_prep_slave_sg()
1144 	edesc->residue_stat = edesc->residue;  in edma_prep_slave_sg()
1146 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_slave_sg()
1154 	struct edma_desc *edesc;  in edma_prep_dma_memcpy()  local
1205 	edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);  in edma_prep_dma_memcpy()
1206 	if (!edesc)  in edma_prep_dma_memcpy()
1209 	edesc->pset_nr = nslots;  in edma_prep_dma_memcpy()
1210 	edesc->residue = edesc->residue_stat = len;  in edma_prep_dma_memcpy()
1211 	edesc->direction = DMA_MEM_TO_MEM;  in edma_prep_dma_memcpy()
1212 	edesc->echan = echan;  in edma_prep_dma_memcpy()
1214 	ret = edma_config_pset(chan, &edesc->pset[0], src, dest, 1,  in edma_prep_dma_memcpy()
1217 		kfree(edesc);  in edma_prep_dma_memcpy()
1221 	edesc->absync = ret;  in edma_prep_dma_memcpy()
1223 	edesc->pset[0].param.opt |= ITCCHEN;  in edma_prep_dma_memcpy()
1227 			edesc->pset[0].param.opt |= TCINTEN;  in edma_prep_dma_memcpy()
1230 		edesc->pset[0].param.opt |= TCCHEN;  in edma_prep_dma_memcpy()
1236 				kfree(edesc);  in edma_prep_dma_memcpy()
1246 		ret = edma_config_pset(chan, &edesc->pset[1], src, dest, 1,  in edma_prep_dma_memcpy()
1249 			kfree(edesc);  in edma_prep_dma_memcpy()
1253 		edesc->pset[1].param.opt |= ITCCHEN;  in edma_prep_dma_memcpy()
1256 			edesc->pset[1].param.opt |= TCINTEN;  in edma_prep_dma_memcpy()
1260 		edesc->polled = true;  in edma_prep_dma_memcpy()
1262 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_dma_memcpy()
1272 	struct edma_desc *edesc;  in edma_prep_dma_cyclic()  local
1333 	edesc = kzalloc(struct_size(edesc, pset, nslots), GFP_ATOMIC);  in edma_prep_dma_cyclic()
1334 	if (!edesc)  in edma_prep_dma_cyclic()
1337 	edesc->cyclic = 1;  in edma_prep_dma_cyclic()
1338 	edesc->pset_nr = nslots;  in edma_prep_dma_cyclic()
1339 	edesc->residue = edesc->residue_stat = buf_len;  in edma_prep_dma_cyclic()
1340 	edesc->direction = direction;  in edma_prep_dma_cyclic()
1341 	edesc->echan = echan;  in edma_prep_dma_cyclic()
1352 				kfree(edesc);  in edma_prep_dma_cyclic()
1360 			memcpy(&edesc->pset[i], &edesc->pset[0],  in edma_prep_dma_cyclic()
1361 			       sizeof(edesc->pset[0]));  in edma_prep_dma_cyclic()
1365 		ret = edma_config_pset(chan, &edesc->pset[i], src_addr,  in edma_prep_dma_cyclic()
1369 			kfree(edesc);  in edma_prep_dma_cyclic()
1392 			edesc->pset[i].param.opt,  in edma_prep_dma_cyclic()
1393 			edesc->pset[i].param.src,  in edma_prep_dma_cyclic()
1394 			edesc->pset[i].param.dst,  in edma_prep_dma_cyclic()
1395 			edesc->pset[i].param.a_b_cnt,  in edma_prep_dma_cyclic()
1396 			edesc->pset[i].param.ccnt,  in edma_prep_dma_cyclic()
1397 			edesc->pset[i].param.src_dst_bidx,  in edma_prep_dma_cyclic()
1398 			edesc->pset[i].param.src_dst_cidx,  in edma_prep_dma_cyclic()
1399 			edesc->pset[i].param.link_bcntrld);  in edma_prep_dma_cyclic()
1401 		edesc->absync = ret;  in edma_prep_dma_cyclic()
1407 			edesc->pset[i].param.opt |= TCINTEN;  in edma_prep_dma_cyclic()
1411 				edesc->pset[i].param.opt |= ITCINTEN;  in edma_prep_dma_cyclic()
1419 	return vchan_tx_prep(&echan->vchan, &edesc->vdesc, tx_flags);  in edma_prep_dma_cyclic()
1425 	struct edma_desc *edesc;  in edma_completion_handler()  local
1428 	edesc = echan->edesc;  in edma_completion_handler()
1429 	if (edesc) {  in edma_completion_handler()
1430 		if (edesc->cyclic) {  in edma_completion_handler()
1431 			vchan_cyclic_callback(&edesc->vdesc);  in edma_completion_handler()
1434 		} else if (edesc->processed == edesc->pset_nr) {  in edma_completion_handler()
1435 			edesc->residue = 0;  in edma_completion_handler()
1437 			vchan_cookie_complete(&edesc->vdesc);  in edma_completion_handler()
1438 			echan->edesc = NULL;  in edma_completion_handler()
1449 			edesc->residue -= edesc->sg_len;  in edma_completion_handler()
1450 			edesc->residue_stat = edesc->residue;  in edma_completion_handler()
1451 			edesc->processed_stat = edesc->processed;  in edma_completion_handler()
1512 	if (!echan->edesc)  in edma_error_handler()
1721 	if (vchan_issue_pending(&echan->vchan) && !echan->edesc)  in edma_issue_pending()
1735 static u32 edma_residue(struct edma_desc *edesc)  in edma_residue()  argument
1737 	bool dst = edesc->direction == DMA_DEV_TO_MEM;  in edma_residue()
1739 	struct edma_chan *echan = edesc->echan;  in edma_residue()
1740 	struct edma_pset *pset = edesc->pset;  in edma_residue()
1762 	if (is_slave_direction(edesc->direction))  in edma_residue()
1790 	if (edesc->cyclic) {  in edma_residue()
1792 		edesc->residue_stat = edesc->residue - done;  in edma_residue()
1793 		return edesc->residue_stat;  in edma_residue()
1806 	pset += edesc->processed_stat;  in edma_residue()
1808 	for (i = edesc->processed_stat; i < edesc->processed; i++, pset++) {  in edma_residue()
1815 			return edesc->residue_stat - (pos - pset->addr);  in edma_residue()
1818 		edesc->processed_stat++;  in edma_residue()
1819 		edesc->residue_stat -= pset->len;  in edma_residue()
1821 	return edesc->residue_stat;  in edma_residue()
1844 	if (echan->edesc && echan->edesc->vdesc.tx.cookie == cookie) {  in edma_tx_status()
1845 		txstate->residue = edma_residue(echan->edesc);  in edma_tx_status()
1861 	    echan->edesc && echan->edesc->polled &&  in edma_tx_status()
1862 	    echan->edesc->vdesc.tx.cookie == cookie) {  in edma_tx_status()
1864 		vchan_cookie_complete(&echan->edesc->vdesc);  in edma_tx_status()
1865 		echan->edesc = NULL;  in edma_tx_status()