145 #define ZYNQMP_DMA_DESC_SIZE(chan)	(chan->desc_size)  argument
147 #define to_chan(chan)		container_of(chan, struct zynqmp_dma_chan, \  argument
255 	struct zynqmp_dma_chan *chan;  member
260 static inline void zynqmp_dma_writeq(struct zynqmp_dma_chan *chan, u32 reg,  in zynqmp_dma_writeq()  argument
263 	lo_hi_writeq(value, chan->regs + reg);  in zynqmp_dma_writeq()
271 static void zynqmp_dma_update_desc_to_ctrlr(struct zynqmp_dma_chan *chan,  in zynqmp_dma_update_desc_to_ctrlr()  argument
277 	zynqmp_dma_writeq(chan, ZYNQMP_DMA_SRC_START_LSB, addr);  in zynqmp_dma_update_desc_to_ctrlr()
279 	zynqmp_dma_writeq(chan, ZYNQMP_DMA_DST_START_LSB, addr);  in zynqmp_dma_update_desc_to_ctrlr()
287 static void zynqmp_dma_desc_config_eod(struct zynqmp_dma_chan *chan,  in zynqmp_dma_desc_config_eod()  argument
306 static void zynqmp_dma_config_sg_ll_desc(struct zynqmp_dma_chan *chan,  in zynqmp_dma_config_sg_ll_desc()  argument
318 	if (chan->is_dmacoherent) {  in zynqmp_dma_config_sg_ll_desc()
324 		dma_addr_t addr = chan->desc_pool_p +  in zynqmp_dma_config_sg_ll_desc()
325 			    ((uintptr_t)sdesc - (uintptr_t)chan->desc_pool_v);  in zynqmp_dma_config_sg_ll_desc()
328 		ddesc->nxtdscraddr = addr + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_config_sg_ll_desc()
336 static void zynqmp_dma_init(struct zynqmp_dma_chan *chan)  in zynqmp_dma_init()  argument
340 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_init()
341 	val = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
342 	writel(val, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_init()
344 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
348 		writel(val, chan->regs + ZYNQMP_DMA_DSCR_ATTR);  in zynqmp_dma_init()
351 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
352 	if (chan->is_dmacoherent) {  in zynqmp_dma_init()
358 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_init()
361 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_init()
362 	val = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_init()
364 	chan->idle = true;  in zynqmp_dma_init()
375 	struct zynqmp_dma_chan *chan = to_chan(tx->chan);  in zynqmp_dma_tx_submit()  local
380 	spin_lock_bh(&chan->lock);  in zynqmp_dma_tx_submit()
383 	if (!list_empty(&chan->pending_list)) {  in zynqmp_dma_tx_submit()
384 		desc = list_last_entry(&chan->pending_list,  in zynqmp_dma_tx_submit()
395 	list_add_tail(&new->node, &chan->pending_list);  in zynqmp_dma_tx_submit()
396 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_tx_submit()
408 zynqmp_dma_get_descriptor(struct zynqmp_dma_chan *chan)  in zynqmp_dma_get_descriptor()  argument
412 	spin_lock_bh(&chan->lock);  in zynqmp_dma_get_descriptor()
413 	desc = list_first_entry(&chan->free_list,  in zynqmp_dma_get_descriptor()
416 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_get_descriptor()
420 	memset((void *)desc->src_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
421 	memset((void *)desc->dst_v, 0, ZYNQMP_DMA_DESC_SIZE(chan));  in zynqmp_dma_get_descriptor()
431 static void zynqmp_dma_free_descriptor(struct zynqmp_dma_chan *chan,  in zynqmp_dma_free_descriptor()  argument
436 	chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
437 	list_add_tail(&sdesc->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
439 		chan->desc_free_cnt++;  in zynqmp_dma_free_descriptor()
440 		list_move_tail(&child->node, &chan->free_list);  in zynqmp_dma_free_descriptor()
449 static void zynqmp_dma_free_desc_list(struct zynqmp_dma_chan *chan,  in zynqmp_dma_free_desc_list()  argument
455 		zynqmp_dma_free_descriptor(chan, desc);  in zynqmp_dma_free_desc_list()
466 	struct zynqmp_dma_chan *chan = to_chan(dchan);  in zynqmp_dma_alloc_chan_resources()  local
470 	ret = pm_runtime_get_sync(chan->dev);  in zynqmp_dma_alloc_chan_resources()
474 	chan->sw_desc_pool = kcalloc(ZYNQMP_DMA_NUM_DESCS, sizeof(*desc),  in zynqmp_dma_alloc_chan_resources()
476 	if (!chan->sw_desc_pool)  in zynqmp_dma_alloc_chan_resources()
479 	chan->idle = true;  in zynqmp_dma_alloc_chan_resources()
480 	chan->desc_free_cnt = ZYNQMP_DMA_NUM_DESCS;  in zynqmp_dma_alloc_chan_resources()
482 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_alloc_chan_resources()
485 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
486 		dma_async_tx_descriptor_init(&desc->async_tx, &chan->common);  in zynqmp_dma_alloc_chan_resources()
488 		list_add_tail(&desc->node, &chan->free_list);  in zynqmp_dma_alloc_chan_resources()
491 	chan->desc_pool_v = dma_zalloc_coherent(chan->dev,  in zynqmp_dma_alloc_chan_resources()
492 				(2 * chan->desc_size * ZYNQMP_DMA_NUM_DESCS),  in zynqmp_dma_alloc_chan_resources()
493 				&chan->desc_pool_p, GFP_KERNEL);  in zynqmp_dma_alloc_chan_resources()
494 	if (!chan->desc_pool_v)  in zynqmp_dma_alloc_chan_resources()
498 		desc = chan->sw_desc_pool + i;  in zynqmp_dma_alloc_chan_resources()
499 		desc->src_v = (struct zynqmp_dma_desc_ll *) (chan->desc_pool_v +  in zynqmp_dma_alloc_chan_resources()
500 					(i * ZYNQMP_DMA_DESC_SIZE(chan) * 2));  in zynqmp_dma_alloc_chan_resources()
502 		desc->src_p = chan->desc_pool_p +  in zynqmp_dma_alloc_chan_resources()
503 				(i * ZYNQMP_DMA_DESC_SIZE(chan) * 2);  in zynqmp_dma_alloc_chan_resources()
504 		desc->dst_p = desc->src_p + ZYNQMP_DMA_DESC_SIZE(chan);  in zynqmp_dma_alloc_chan_resources()
514 static void zynqmp_dma_start(struct zynqmp_dma_chan *chan)  in zynqmp_dma_start()  argument
516 	writel(ZYNQMP_DMA_INT_EN_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IER);  in zynqmp_dma_start()
517 	writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_start()
518 	chan->idle = false;  in zynqmp_dma_start()
519 	writel(ZYNQMP_DMA_ENABLE, chan->regs + ZYNQMP_DMA_CTRL2);  in zynqmp_dma_start()
527 static void zynqmp_dma_handle_ovfl_int(struct zynqmp_dma_chan *chan, u32 status)  in zynqmp_dma_handle_ovfl_int()  argument
530 		writel(0, chan->regs + ZYNQMP_DMA_TOTAL_BYTE);  in zynqmp_dma_handle_ovfl_int()
532 		readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_handle_ovfl_int()
534 		readl(chan->regs + ZYNQMP_DMA_IRQ_SRC_ACCT);  in zynqmp_dma_handle_ovfl_int()
537 static void zynqmp_dma_config(struct zynqmp_dma_chan *chan)  in zynqmp_dma_config()  argument
541 	val = readl(chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
543 	writel(val, chan->regs + ZYNQMP_DMA_CTRL0);  in zynqmp_dma_config()
545 	val = readl(chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
547 		(chan->src_burst_len << ZYNQMP_DMA_ARLEN_OFST);  in zynqmp_dma_config()
549 		(chan->dst_burst_len << ZYNQMP_DMA_AWLEN_OFST);  in zynqmp_dma_config()
550 	writel(val, chan->regs + ZYNQMP_DMA_DATA_ATTR);  in zynqmp_dma_config()
563 	struct zynqmp_dma_chan *chan = to_chan(dchan);  in zynqmp_dma_device_config()  local
565 	chan->src_burst_len = config->src_maxburst;  in zynqmp_dma_device_config()
566 	chan->dst_burst_len = config->dst_maxburst;  in zynqmp_dma_device_config()
575 static void zynqmp_dma_start_transfer(struct zynqmp_dma_chan *chan)  in zynqmp_dma_start_transfer()  argument
579 	if (!chan->idle)  in zynqmp_dma_start_transfer()
582 	zynqmp_dma_config(chan);  in zynqmp_dma_start_transfer()
584 	desc = list_first_entry_or_null(&chan->pending_list,  in zynqmp_dma_start_transfer()
589 	list_splice_tail_init(&chan->pending_list, &chan->active_list);  in zynqmp_dma_start_transfer()
590 	zynqmp_dma_update_desc_to_ctrlr(chan, desc);  in zynqmp_dma_start_transfer()
591 	zynqmp_dma_start(chan);  in zynqmp_dma_start_transfer()
599 static void zynqmp_dma_chan_desc_cleanup(struct zynqmp_dma_chan *chan)  in zynqmp_dma_chan_desc_cleanup()  argument
603 	list_for_each_entry_safe(desc, next, &chan->done_list, node) {  in zynqmp_dma_chan_desc_cleanup()
612 			spin_unlock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
614 			spin_lock(&chan->lock);  in zynqmp_dma_chan_desc_cleanup()
618 		zynqmp_dma_free_descriptor(chan, desc);  in zynqmp_dma_chan_desc_cleanup()
626 static void zynqmp_dma_complete_descriptor(struct zynqmp_dma_chan *chan)  in zynqmp_dma_complete_descriptor()  argument
630 	desc = list_first_entry_or_null(&chan->active_list,  in zynqmp_dma_complete_descriptor()
636 	list_add_tail(&desc->node, &chan->done_list);  in zynqmp_dma_complete_descriptor()
645 	struct zynqmp_dma_chan *chan = to_chan(dchan);  in zynqmp_dma_issue_pending()  local
647 	spin_lock_bh(&chan->lock);  in zynqmp_dma_issue_pending()
648 	zynqmp_dma_start_transfer(chan);  in zynqmp_dma_issue_pending()
649 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_issue_pending()
656 static void zynqmp_dma_free_descriptors(struct zynqmp_dma_chan *chan)  in zynqmp_dma_free_descriptors()  argument
658 	zynqmp_dma_free_desc_list(chan, &chan->active_list);  in zynqmp_dma_free_descriptors()
659 	zynqmp_dma_free_desc_list(chan, &chan->pending_list);  in zynqmp_dma_free_descriptors()
660 	zynqmp_dma_free_desc_list(chan, &chan->done_list);  in zynqmp_dma_free_descriptors()
669 	struct zynqmp_dma_chan *chan = to_chan(dchan);  in zynqmp_dma_free_chan_resources()  local
671 	spin_lock_bh(&chan->lock);  in zynqmp_dma_free_chan_resources()
672 	zynqmp_dma_free_descriptors(chan);  in zynqmp_dma_free_chan_resources()
673 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_free_chan_resources()
674 	dma_free_coherent(chan->dev,  in zynqmp_dma_free_chan_resources()
675 		(2 * ZYNQMP_DMA_DESC_SIZE(chan) * ZYNQMP_DMA_NUM_DESCS),  in zynqmp_dma_free_chan_resources()
676 		chan->desc_pool_v, chan->desc_pool_p);  in zynqmp_dma_free_chan_resources()
677 	kfree(chan->sw_desc_pool);  in zynqmp_dma_free_chan_resources()
678 	pm_runtime_mark_last_busy(chan->dev);  in zynqmp_dma_free_chan_resources()
679 	pm_runtime_put_autosuspend(chan->dev);  in zynqmp_dma_free_chan_resources()
686 static void zynqmp_dma_reset(struct zynqmp_dma_chan *chan)  in zynqmp_dma_reset()  argument
688 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_reset()
690 	zynqmp_dma_complete_descriptor(chan);  in zynqmp_dma_reset()
691 	zynqmp_dma_chan_desc_cleanup(chan);  in zynqmp_dma_reset()
692 	zynqmp_dma_free_descriptors(chan);  in zynqmp_dma_reset()
693 	zynqmp_dma_init(chan);  in zynqmp_dma_reset()
705 	struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data;  in zynqmp_dma_irq_handler()  local
709 	isr = readl(chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
710 	imr = readl(chan->regs + ZYNQMP_DMA_IMR);  in zynqmp_dma_irq_handler()
713 	writel(isr, chan->regs + ZYNQMP_DMA_ISR);  in zynqmp_dma_irq_handler()
715 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
720 		chan->idle = true;  in zynqmp_dma_irq_handler()
723 		chan->err = true;  in zynqmp_dma_irq_handler()
724 		tasklet_schedule(&chan->tasklet);  in zynqmp_dma_irq_handler()
725 		dev_err(chan->dev, "Channel %p has errors\n", chan);  in zynqmp_dma_irq_handler()
730 		zynqmp_dma_handle_ovfl_int(chan, status);  in zynqmp_dma_irq_handler()
731 		dev_dbg(chan->dev, "Channel %p overflow interrupt\n", chan);  in zynqmp_dma_irq_handler()
744 	struct zynqmp_dma_chan *chan = (struct zynqmp_dma_chan *)data;  in zynqmp_dma_do_tasklet()  local
747 	spin_lock(&chan->lock);  in zynqmp_dma_do_tasklet()
749 	if (chan->err) {  in zynqmp_dma_do_tasklet()
750 		zynqmp_dma_reset(chan);  in zynqmp_dma_do_tasklet()
751 		chan->err = false;  in zynqmp_dma_do_tasklet()
755 	count = readl(chan->regs + ZYNQMP_DMA_IRQ_DST_ACCT);  in zynqmp_dma_do_tasklet()
758 		zynqmp_dma_complete_descriptor(chan);  in zynqmp_dma_do_tasklet()
759 		zynqmp_dma_chan_desc_cleanup(chan);  in zynqmp_dma_do_tasklet()
763 	if (chan->idle)  in zynqmp_dma_do_tasklet()
764 		zynqmp_dma_start_transfer(chan);  in zynqmp_dma_do_tasklet()
767 	spin_unlock(&chan->lock);  in zynqmp_dma_do_tasklet()
778 	struct zynqmp_dma_chan *chan = to_chan(dchan);  in zynqmp_dma_device_terminate_all()  local
780 	spin_lock_bh(&chan->lock);  in zynqmp_dma_device_terminate_all()
781 	writel(ZYNQMP_DMA_IDS_DEFAULT_MASK, chan->regs + ZYNQMP_DMA_IDS);  in zynqmp_dma_device_terminate_all()
782 	zynqmp_dma_free_descriptors(chan);  in zynqmp_dma_device_terminate_all()
783 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_device_terminate_all()
802 	struct zynqmp_dma_chan *chan;  in zynqmp_dma_prep_memcpy()  local
808 	chan = to_chan(dchan);  in zynqmp_dma_prep_memcpy()
812 	spin_lock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
813 	if (desc_cnt > chan->desc_free_cnt) {  in zynqmp_dma_prep_memcpy()
814 		spin_unlock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
815 		dev_dbg(chan->dev, "chan %p descs are not available\n", chan);  in zynqmp_dma_prep_memcpy()
818 	chan->desc_free_cnt = chan->desc_free_cnt - desc_cnt;  in zynqmp_dma_prep_memcpy()
819 	spin_unlock_bh(&chan->lock);  in zynqmp_dma_prep_memcpy()
823 		new = zynqmp_dma_get_descriptor(chan);  in zynqmp_dma_prep_memcpy()
827 		zynqmp_dma_config_sg_ll_desc(chan, desc, dma_src,  in zynqmp_dma_prep_memcpy()
839 	zynqmp_dma_desc_config_eod(chan, desc);  in zynqmp_dma_prep_memcpy()
849 static void zynqmp_dma_chan_remove(struct zynqmp_dma_chan *chan)  in zynqmp_dma_chan_remove()  argument
851 	if (!chan)  in zynqmp_dma_chan_remove()
854 	if (chan->irq)  in zynqmp_dma_chan_remove()
855 		devm_free_irq(chan->zdev->dev, chan->irq, chan);  in zynqmp_dma_chan_remove()
856 	tasklet_kill(&chan->tasklet);  in zynqmp_dma_chan_remove()
857 	list_del(&chan->common.device_node);  in zynqmp_dma_chan_remove()
870 	struct zynqmp_dma_chan *chan;  in zynqmp_dma_chan_probe()  local
875 	chan = devm_kzalloc(zdev->dev, sizeof(*chan), GFP_KERNEL);  in zynqmp_dma_chan_probe()
876 	if (!chan)  in zynqmp_dma_chan_probe()
878 	chan->dev = zdev->dev;  in zynqmp_dma_chan_probe()
879 	chan->zdev = zdev;  in zynqmp_dma_chan_probe()
882 	chan->regs = devm_ioremap_resource(&pdev->dev, res);  in zynqmp_dma_chan_probe()
883 	if (IS_ERR(chan->regs))  in zynqmp_dma_chan_probe()
884 		return PTR_ERR(chan->regs);  in zynqmp_dma_chan_probe()
886 	chan->bus_width = ZYNQMP_DMA_BUS_WIDTH_64;  in zynqmp_dma_chan_probe()
887 	chan->dst_burst_len = ZYNQMP_DMA_AWLEN_RST_VAL;  in zynqmp_dma_chan_probe()
888 	chan->src_burst_len = ZYNQMP_DMA_ARLEN_RST_VAL;  in zynqmp_dma_chan_probe()
889 	err = of_property_read_u32(node, "xlnx,bus-width", &chan->bus_width);  in zynqmp_dma_chan_probe()
895 	if (chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_64 &&  in zynqmp_dma_chan_probe()
896 	    chan->bus_width != ZYNQMP_DMA_BUS_WIDTH_128) {  in zynqmp_dma_chan_probe()
901 	chan->is_dmacoherent =  of_property_read_bool(node, "dma-coherent");  in zynqmp_dma_chan_probe()
902 	zdev->chan = chan;  in zynqmp_dma_chan_probe()
903 	tasklet_init(&chan->tasklet, zynqmp_dma_do_tasklet, (ulong)chan);  in zynqmp_dma_chan_probe()
904 	spin_lock_init(&chan->lock);  in zynqmp_dma_chan_probe()
905 	INIT_LIST_HEAD(&chan->active_list);  in zynqmp_dma_chan_probe()
906 	INIT_LIST_HEAD(&chan->pending_list);  in zynqmp_dma_chan_probe()
907 	INIT_LIST_HEAD(&chan->done_list);  in zynqmp_dma_chan_probe()
908 	INIT_LIST_HEAD(&chan->free_list);  in zynqmp_dma_chan_probe()
910 	dma_cookie_init(&chan->common);  in zynqmp_dma_chan_probe()
911 	chan->common.device = &zdev->common;  in zynqmp_dma_chan_probe()
912 	list_add_tail(&chan->common.device_node, &zdev->common.channels);  in zynqmp_dma_chan_probe()
914 	zynqmp_dma_init(chan);  in zynqmp_dma_chan_probe()
915 	chan->irq = platform_get_irq(pdev, 0);  in zynqmp_dma_chan_probe()
916 	if (chan->irq < 0)  in zynqmp_dma_chan_probe()
918 	err = devm_request_irq(&pdev->dev, chan->irq, zynqmp_dma_irq_handler, 0,  in zynqmp_dma_chan_probe()
919 			       "zynqmp-dma", chan);  in zynqmp_dma_chan_probe()
923 	chan->desc_size = sizeof(struct zynqmp_dma_desc_ll);  in zynqmp_dma_chan_probe()
924 	chan->idle = true;  in zynqmp_dma_chan_probe()
940 	return dma_get_slave_channel(&zdev->chan->common);  in of_zynqmp_dma_xlate()
1085 	p->dst_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1086 	p->src_addr_widths = BIT(zdev->chan->bus_width / 8);  in zynqmp_dma_probe()
1106 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_probe()
1127 	zynqmp_dma_chan_remove(zdev->chan);  in zynqmp_dma_remove()