253 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(c);  in dma_chan_to_mpc_dma()  local
255 	return container_of(mchan, struct mpc_dma, channels[c->chan_id]);  in dma_chan_to_mpc_dma()
266 static void mpc_dma_execute(struct mpc_dma_chan *mchan)  in mpc_dma_execute()  argument
268 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);  in mpc_dma_execute()
272 	int cid = mchan->chan.chan_id;  in mpc_dma_execute()
274 	while (!list_empty(&mchan->queued)) {  in mpc_dma_execute()
275 		mdesc = list_first_entry(&mchan->queued,  in mpc_dma_execute()
284 			if (list_empty(&mchan->active))  in mpc_dma_execute()
285 				list_move_tail(&mdesc->node, &mchan->active);  in mpc_dma_execute()
288 			list_move_tail(&mdesc->node, &mchan->active);  in mpc_dma_execute()
293 	list_for_each_entry(mdesc, &mchan->active, node) {  in mpc_dma_execute()
332 	struct mpc_dma_chan *mchan;  in mpc_dma_irq_process()  local
339 		mchan = &mdma->channels[ch + off];  in mpc_dma_irq_process()
341 		spin_lock(&mchan->lock);  in mpc_dma_irq_process()
348 			list_for_each_entry(mdesc, &mchan->active, node)  in mpc_dma_irq_process()
352 		list_splice_tail_init(&mchan->active, &mchan->completed);  in mpc_dma_irq_process()
353 		if (!list_empty(&mchan->queued))  in mpc_dma_irq_process()
354 			mpc_dma_execute(mchan);  in mpc_dma_irq_process()
356 		spin_unlock(&mchan->lock);  in mpc_dma_irq_process()
391 	struct mpc_dma_chan *mchan;  in mpc_dma_process_completed()  local
399 		mchan = &mdma->channels[i];  in mpc_dma_process_completed()
402 		spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_process_completed()
403 		if (!list_empty(&mchan->completed))  in mpc_dma_process_completed()
404 			list_splice_tail_init(&mchan->completed, &list);  in mpc_dma_process_completed()
405 		spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_process_completed()
421 		spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_process_completed()
422 		list_splice_tail_init(&list, &mchan->free);  in mpc_dma_process_completed()
423 		mchan->chan.completed_cookie = last_cookie;  in mpc_dma_process_completed()
424 		spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_process_completed()
474 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(txd->chan);  in mpc_dma_tx_submit()  local
481 	spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_tx_submit()
484 	list_move_tail(&mdesc->node, &mchan->queued);  in mpc_dma_tx_submit()
487 	if (list_empty(&mchan->active))  in mpc_dma_tx_submit()
488 		mpc_dma_execute(mchan);  in mpc_dma_tx_submit()
492 	spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_tx_submit()
501 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_alloc_chan_resources()  local
543 	spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_alloc_chan_resources()
544 	mchan->tcd = tcd;  in mpc_dma_alloc_chan_resources()
545 	mchan->tcd_paddr = tcd_paddr;  in mpc_dma_alloc_chan_resources()
546 	list_splice_tail_init(&descs, &mchan->free);  in mpc_dma_alloc_chan_resources()
547 	spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_alloc_chan_resources()
559 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_free_chan_resources()  local
566 	spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_free_chan_resources()
569 	BUG_ON(!list_empty(&mchan->prepared));  in mpc_dma_free_chan_resources()
570 	BUG_ON(!list_empty(&mchan->queued));  in mpc_dma_free_chan_resources()
571 	BUG_ON(!list_empty(&mchan->active));  in mpc_dma_free_chan_resources()
572 	BUG_ON(!list_empty(&mchan->completed));  in mpc_dma_free_chan_resources()
575 	list_splice_tail_init(&mchan->free, &descs);  in mpc_dma_free_chan_resources()
576 	tcd = mchan->tcd;  in mpc_dma_free_chan_resources()
577 	tcd_paddr = mchan->tcd_paddr;  in mpc_dma_free_chan_resources()
579 	spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_free_chan_resources()
617 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_prep_memcpy()  local
623 	spin_lock_irqsave(&mchan->lock, iflags);  in mpc_dma_prep_memcpy()
624 	if (!list_empty(&mchan->free)) {  in mpc_dma_prep_memcpy()
625 		mdesc = list_first_entry(&mchan->free, struct mpc_dma_desc,  in mpc_dma_prep_memcpy()
629 	spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_memcpy()
679 	spin_lock_irqsave(&mchan->lock, iflags);  in mpc_dma_prep_memcpy()
680 	list_add_tail(&mdesc->node, &mchan->prepared);  in mpc_dma_prep_memcpy()
681 	spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_memcpy()
701 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_prep_slave_sg()  local
719 		spin_lock_irqsave(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
721 		mdesc = list_first_entry(&mchan->free,  in mpc_dma_prep_slave_sg()
724 			spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
733 			per_paddr = mchan->src_per_paddr;  in mpc_dma_prep_slave_sg()
734 			tcd_nunits = mchan->src_tcd_nunits;  in mpc_dma_prep_slave_sg()
736 			per_paddr = mchan->dst_per_paddr;  in mpc_dma_prep_slave_sg()
737 			tcd_nunits = mchan->dst_tcd_nunits;  in mpc_dma_prep_slave_sg()
740 		spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
757 			if (!IS_ALIGNED(sg_dma_address(sg), mchan->dwidth))  in mpc_dma_prep_slave_sg()
761 			tcd->doff = mchan->dwidth;  in mpc_dma_prep_slave_sg()
766 			if (!IS_ALIGNED(sg_dma_address(sg), mchan->swidth))  in mpc_dma_prep_slave_sg()
769 			tcd->soff = mchan->swidth;  in mpc_dma_prep_slave_sg()
773 		tcd->ssize = buswidth_to_dmatsize(mchan->swidth);  in mpc_dma_prep_slave_sg()
774 		tcd->dsize = buswidth_to_dmatsize(mchan->dwidth);  in mpc_dma_prep_slave_sg()
778 			if (!IS_ALIGNED(tcd->nbytes, mchan->swidth))  in mpc_dma_prep_slave_sg()
806 		spin_lock_irqsave(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
807 		list_add_tail(&mdesc->node, &mchan->prepared);  in mpc_dma_prep_slave_sg()
808 		spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
815 	spin_lock_irqsave(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
816 	list_add_tail(&mdesc->node, &mchan->free);  in mpc_dma_prep_slave_sg()
817 	spin_unlock_irqrestore(&mchan->lock, iflags);  in mpc_dma_prep_slave_sg()
843 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_device_config()  local
844 	struct mpc_dma *mdma = dma_chan_to_mpc_dma(&mchan->chan);  in mpc_dma_device_config()
870 	spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_device_config()
872 	mchan->src_per_paddr = cfg->src_addr;  in mpc_dma_device_config()
873 	mchan->src_tcd_nunits = cfg->src_maxburst;  in mpc_dma_device_config()
874 	mchan->swidth = cfg->src_addr_width;  in mpc_dma_device_config()
875 	mchan->dst_per_paddr = cfg->dst_addr;  in mpc_dma_device_config()
876 	mchan->dst_tcd_nunits = cfg->dst_maxburst;  in mpc_dma_device_config()
877 	mchan->dwidth = cfg->dst_addr_width;  in mpc_dma_device_config()
880 	if (mchan->src_tcd_nunits == 0)  in mpc_dma_device_config()
881 		mchan->src_tcd_nunits = 1;  in mpc_dma_device_config()
882 	if (mchan->dst_tcd_nunits == 0)  in mpc_dma_device_config()
883 		mchan->dst_tcd_nunits = 1;  in mpc_dma_device_config()
885 	spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_device_config()
892 	struct mpc_dma_chan *mchan = dma_chan_to_mpc_dma_chan(chan);  in mpc_dma_device_terminate_all()  local
897 	spin_lock_irqsave(&mchan->lock, flags);  in mpc_dma_device_terminate_all()
900 	list_splice_tail_init(&mchan->prepared, &mchan->free);  in mpc_dma_device_terminate_all()
901 	list_splice_tail_init(&mchan->queued, &mchan->free);  in mpc_dma_device_terminate_all()
902 	list_splice_tail_init(&mchan->active, &mchan->free);  in mpc_dma_device_terminate_all()
904 	spin_unlock_irqrestore(&mchan->lock, flags);  in mpc_dma_device_terminate_all()
915 	struct mpc_dma_chan *mchan;  in mpc_dma_probe()  local
1009 		mchan = &mdma->channels[i];  in mpc_dma_probe()
1011 		mchan->chan.device = dma;  in mpc_dma_probe()
1012 		dma_cookie_init(&mchan->chan);  in mpc_dma_probe()
1014 		INIT_LIST_HEAD(&mchan->free);  in mpc_dma_probe()
1015 		INIT_LIST_HEAD(&mchan->prepared);  in mpc_dma_probe()
1016 		INIT_LIST_HEAD(&mchan->queued);  in mpc_dma_probe()
1017 		INIT_LIST_HEAD(&mchan->active);  in mpc_dma_probe()
1018 		INIT_LIST_HEAD(&mchan->completed);  in mpc_dma_probe()
1020 		spin_lock_init(&mchan->lock);  in mpc_dma_probe()
1021 		list_add_tail(&mchan->chan.device_node, &dma->channels);  in mpc_dma_probe()