242 static inline bool imxdma_chan_is_doing_cyclic(struct imxdma_channel *imxdmac)  in imxdma_chan_is_doing_cyclic()  argument
246 	if (!list_empty(&imxdmac->ld_active)) {  in imxdma_chan_is_doing_cyclic()
247 		desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc,  in imxdma_chan_is_doing_cyclic()
268 static int imxdma_hw_chain(struct imxdma_channel *imxdmac)  in imxdma_hw_chain()  argument
270 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_hw_chain()
273 		return imxdmac->hw_chaining;  in imxdma_hw_chain()
283 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_sg_next()  local
284 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_sg_next()
294 				 DMA_DAR(imxdmac->channel));  in imxdma_sg_next()
297 				 DMA_SAR(imxdmac->channel));  in imxdma_sg_next()
299 	imx_dmav1_writel(imxdma, now, DMA_CNTR(imxdmac->channel));  in imxdma_sg_next()
302 		"size 0x%08x\n", __func__, imxdmac->channel,  in imxdma_sg_next()
303 		 imx_dmav1_readl(imxdma, DMA_DAR(imxdmac->channel)),  in imxdma_sg_next()
304 		 imx_dmav1_readl(imxdma, DMA_SAR(imxdmac->channel)),  in imxdma_sg_next()
305 		 imx_dmav1_readl(imxdma, DMA_CNTR(imxdmac->channel)));  in imxdma_sg_next()
310 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_enable_hw()  local
311 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_enable_hw()
312 	int channel = imxdmac->channel;  in imxdma_enable_hw()
326 			d->sg && imxdma_hw_chain(imxdmac)) {  in imxdma_enable_hw()
340 static void imxdma_disable_hw(struct imxdma_channel *imxdmac)  in imxdma_disable_hw()  argument
342 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_disable_hw()
343 	int channel = imxdmac->channel;  in imxdma_disable_hw()
348 	if (imxdma_hw_chain(imxdmac))  in imxdma_disable_hw()
349 		del_timer(&imxdmac->watchdog);  in imxdma_disable_hw()
362 	struct imxdma_channel *imxdmac = from_timer(imxdmac, t, watchdog);  in imxdma_watchdog()  local
363 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_watchdog()
364 	int channel = imxdmac->channel;  in imxdma_watchdog()
369 	tasklet_schedule(&imxdmac->dma_tasklet);  in imxdma_watchdog()
371 		imxdmac->channel);  in imxdma_watchdog()
427 static void dma_irq_handle_channel(struct imxdma_channel *imxdmac)  in dma_irq_handle_channel()  argument
429 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in dma_irq_handle_channel()
430 	int chno = imxdmac->channel;  in dma_irq_handle_channel()
435 	if (list_empty(&imxdmac->ld_active)) {  in dma_irq_handle_channel()
440 	desc = list_first_entry(&imxdmac->ld_active,  in dma_irq_handle_channel()
454 			if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
458 				mod_timer(&imxdmac->watchdog,  in dma_irq_handle_channel()
471 			if (imxdma_chan_is_doing_cyclic(imxdmac))  in dma_irq_handle_channel()
473 				tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
478 		if (imxdma_hw_chain(imxdmac)) {  in dma_irq_handle_channel()
479 			del_timer(&imxdmac->watchdog);  in dma_irq_handle_channel()
487 	tasklet_schedule(&imxdmac->dma_tasklet);  in dma_irq_handle_channel()
513 	struct imxdma_channel *imxdmac = to_imxdma_chan(d->desc.chan);  in imxdma_xfer_desc()  local
514 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_xfer_desc()
539 		imxdmac->slot_2d = slot;  in imxdma_xfer_desc()
540 		imxdmac->enabled_2d = true;  in imxdma_xfer_desc()
561 		imx_dmav1_writel(imxdma, d->src, DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
562 		imx_dmav1_writel(imxdma, d->dest, DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
564 			 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
566 		imx_dmav1_writel(imxdma, d->len, DMA_CNTR(imxdmac->channel));  in imxdma_xfer_desc()
570 			__func__, imxdmac->channel,  in imxdma_xfer_desc()
579 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
580 					 DMA_SAR(imxdmac->channel));  in imxdma_xfer_desc()
581 			imx_dmav1_writel(imxdma, imxdmac->ccr_from_device,  in imxdma_xfer_desc()
582 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
586 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
588 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
590 			imx_dmav1_writel(imxdma, imxdmac->per_address,  in imxdma_xfer_desc()
591 					 DMA_DAR(imxdmac->channel));  in imxdma_xfer_desc()
592 			imx_dmav1_writel(imxdma, imxdmac->ccr_to_device,  in imxdma_xfer_desc()
593 					 DMA_CCR(imxdmac->channel));  in imxdma_xfer_desc()
597 				__func__, imxdmac->channel,  in imxdma_xfer_desc()
599 				(unsigned long long)imxdmac->per_address);  in imxdma_xfer_desc()
602 				__func__, imxdmac->channel);  in imxdma_xfer_desc()
618 	struct imxdma_channel *imxdmac = (void *)data;  in imxdma_tasklet()  local
619 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tasklet()
625 	if (list_empty(&imxdmac->ld_active)) {  in imxdma_tasklet()
630 	desc = list_first_entry(&imxdmac->ld_active, struct imxdma_desc, node);  in imxdma_tasklet()
636 	if (imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_tasklet()
642 	if (imxdmac->enabled_2d) {  in imxdma_tasklet()
643 		imxdma->slots_2d[imxdmac->slot_2d].count--;  in imxdma_tasklet()
644 		imxdmac->enabled_2d = false;  in imxdma_tasklet()
647 	list_move_tail(imxdmac->ld_active.next, &imxdmac->ld_free);  in imxdma_tasklet()
649 	if (!list_empty(&imxdmac->ld_queue)) {  in imxdma_tasklet()
650 		next_desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_tasklet()
652 		list_move_tail(imxdmac->ld_queue.next, &imxdmac->ld_active);  in imxdma_tasklet()
655 				 __func__, imxdmac->channel);  in imxdma_tasklet()
665 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_terminate_all()  local
666 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_terminate_all()
669 	imxdma_disable_hw(imxdmac);  in imxdma_terminate_all()
672 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_terminate_all()
673 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_terminate_all()
682 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config_write()  local
683 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_config_write()
687 		imxdmac->per_address = dmaengine_cfg->src_addr;  in imxdma_config_write()
688 		imxdmac->watermark_level = dmaengine_cfg->src_maxburst;  in imxdma_config_write()
689 		imxdmac->word_size = dmaengine_cfg->src_addr_width;  in imxdma_config_write()
691 		imxdmac->per_address = dmaengine_cfg->dst_addr;  in imxdma_config_write()
692 		imxdmac->watermark_level = dmaengine_cfg->dst_maxburst;  in imxdma_config_write()
693 		imxdmac->word_size = dmaengine_cfg->dst_addr_width;  in imxdma_config_write()
696 	switch (imxdmac->word_size) {  in imxdma_config_write()
709 	imxdmac->hw_chaining = 0;  in imxdma_config_write()
711 	imxdmac->ccr_from_device = (mode | IMX_DMA_TYPE_FIFO) |  in imxdma_config_write()
714 	imxdmac->ccr_to_device =  in imxdma_config_write()
717 	imx_dmav1_writel(imxdma, imxdmac->dma_request,  in imxdma_config_write()
718 			 DMA_RSSR(imxdmac->channel));  in imxdma_config_write()
721 	imx_dmav1_writel(imxdma, imxdmac->watermark_level *  in imxdma_config_write()
722 			 imxdmac->word_size, DMA_BLR(imxdmac->channel));  in imxdma_config_write()
730 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_config()  local
732 	memcpy(&imxdmac->config, dmaengine_cfg, sizeof(*dmaengine_cfg));  in imxdma_config()
746 	struct imxdma_channel *imxdmac = to_imxdma_chan(tx->chan);  in imxdma_tx_submit()  local
747 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_tx_submit()
752 	list_move_tail(imxdmac->ld_free.next, &imxdmac->ld_queue);  in imxdma_tx_submit()
761 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_alloc_chan_resources()  local
765 		imxdmac->dma_request = data->dma_request;  in imxdma_alloc_chan_resources()
767 	while (imxdmac->descs_allocated < IMXDMA_MAX_CHAN_DESCRIPTORS) {  in imxdma_alloc_chan_resources()
780 		list_add_tail(&desc->node, &imxdmac->ld_free);  in imxdma_alloc_chan_resources()
781 		imxdmac->descs_allocated++;  in imxdma_alloc_chan_resources()
784 	if (!imxdmac->descs_allocated)  in imxdma_alloc_chan_resources()
787 	return imxdmac->descs_allocated;  in imxdma_alloc_chan_resources()
792 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_free_chan_resources()  local
793 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_free_chan_resources()
799 	imxdma_disable_hw(imxdmac);  in imxdma_free_chan_resources()
800 	list_splice_tail_init(&imxdmac->ld_active, &imxdmac->ld_free);  in imxdma_free_chan_resources()
801 	list_splice_tail_init(&imxdmac->ld_queue, &imxdmac->ld_free);  in imxdma_free_chan_resources()
805 	list_for_each_entry_safe(desc, _desc, &imxdmac->ld_free, node) {  in imxdma_free_chan_resources()
807 		imxdmac->descs_allocated--;  in imxdma_free_chan_resources()
809 	INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_free_chan_resources()
811 	kfree(imxdmac->sg_list);  in imxdma_free_chan_resources()
812 	imxdmac->sg_list = NULL;  in imxdma_free_chan_resources()
820 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_slave_sg()  local
825 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_slave_sg()
826 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_slave_sg()
829 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_slave_sg()
835 	switch (imxdmac->word_size) {  in imxdma_prep_slave_sg()
856 		desc->src = imxdmac->per_address;  in imxdma_prep_slave_sg()
858 		desc->dest = imxdmac->per_address;  in imxdma_prep_slave_sg()
871 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_cyclic()  local
872 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_cyclic()
878 			__func__, imxdmac->channel, buf_len, period_len);  in imxdma_prep_dma_cyclic()
880 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_cyclic()
881 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_cyclic()
884 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_cyclic()
886 	kfree(imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
888 	imxdmac->sg_list = kcalloc(periods + 1,  in imxdma_prep_dma_cyclic()
890 	if (!imxdmac->sg_list)  in imxdma_prep_dma_cyclic()
893 	sg_init_table(imxdmac->sg_list, periods);  in imxdma_prep_dma_cyclic()
896 		sg_assign_page(&imxdmac->sg_list[i], NULL);  in imxdma_prep_dma_cyclic()
897 		imxdmac->sg_list[i].offset = 0;  in imxdma_prep_dma_cyclic()
898 		imxdmac->sg_list[i].dma_address = dma_addr;  in imxdma_prep_dma_cyclic()
899 		sg_dma_len(&imxdmac->sg_list[i]) = period_len;  in imxdma_prep_dma_cyclic()
904 	sg_chain(imxdmac->sg_list, periods + 1, imxdmac->sg_list);  in imxdma_prep_dma_cyclic()
907 	desc->sg = imxdmac->sg_list;  in imxdma_prep_dma_cyclic()
912 		desc->src = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
914 		desc->dest = imxdmac->per_address;  in imxdma_prep_dma_cyclic()
919 	imxdma_config_write(chan, &imxdmac->config, direction);  in imxdma_prep_dma_cyclic()
928 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_memcpy()  local
929 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_memcpy()
933 		__func__, imxdmac->channel, (unsigned long long)src,  in imxdma_prep_dma_memcpy()
936 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_memcpy()
937 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_memcpy()
940 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_memcpy()
959 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_prep_dma_interleaved()  local
960 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_prep_dma_interleaved()
965 		imxdmac->channel, (unsigned long long)xt->src_start,  in imxdma_prep_dma_interleaved()
970 	if (list_empty(&imxdmac->ld_free) ||  in imxdma_prep_dma_interleaved()
971 	    imxdma_chan_is_doing_cyclic(imxdmac))  in imxdma_prep_dma_interleaved()
977 	desc = list_first_entry(&imxdmac->ld_free, struct imxdma_desc, node);  in imxdma_prep_dma_interleaved()
1001 	struct imxdma_channel *imxdmac = to_imxdma_chan(chan);  in imxdma_issue_pending()  local
1002 	struct imxdma_engine *imxdma = imxdmac->imxdma;  in imxdma_issue_pending()
1007 	if (list_empty(&imxdmac->ld_active) &&  in imxdma_issue_pending()
1008 	    !list_empty(&imxdmac->ld_queue)) {  in imxdma_issue_pending()
1009 		desc = list_first_entry(&imxdmac->ld_queue,  in imxdma_issue_pending()
1015 				 __func__, imxdmac->channel);  in imxdma_issue_pending()
1017 			list_move_tail(imxdmac->ld_queue.next,  in imxdma_issue_pending()
1018 				       &imxdmac->ld_active);  in imxdma_issue_pending()
1150 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_probe()  local
1162 			imxdmac->irq = irq + i;  in imxdma_probe()
1163 			timer_setup(&imxdmac->watchdog, imxdma_watchdog, 0);  in imxdma_probe()
1166 		imxdmac->imxdma = imxdma;  in imxdma_probe()
1168 		INIT_LIST_HEAD(&imxdmac->ld_queue);  in imxdma_probe()
1169 		INIT_LIST_HEAD(&imxdmac->ld_free);  in imxdma_probe()
1170 		INIT_LIST_HEAD(&imxdmac->ld_active);  in imxdma_probe()
1172 		tasklet_init(&imxdmac->dma_tasklet, imxdma_tasklet,  in imxdma_probe()
1173 			     (unsigned long)imxdmac);  in imxdma_probe()
1174 		imxdmac->chan.device = &imxdma->dma_device;  in imxdma_probe()
1175 		dma_cookie_init(&imxdmac->chan);  in imxdma_probe()
1176 		imxdmac->channel = i;  in imxdma_probe()
1179 		list_add_tail(&imxdmac->chan.device_node,  in imxdma_probe()
1238 		struct imxdma_channel *imxdmac = &imxdma->channel[i];  in imxdma_free_irq()  local
1241 			disable_irq(imxdmac->irq);  in imxdma_free_irq()
1243 		tasklet_kill(&imxdmac->dma_tasklet);  in imxdma_free_irq()