Lines Matching +full:xlnx +full:- +full:zynqmp +full:- +full:dpdma
1 // SPDX-License-Identifier: GPL-2.0
3 * Xilinx ZynqMP DPDMA Engine driver
5 * Copyright (C) 2015 - 2020 Xilinx, Inc.
27 #include <dt-bindings/dma/xlnx-zynqmp-dpdma.h>
30 #include "../virt-dma.h"
32 /* DPDMA registers */
118 /* DPDMA descriptor fields */
141 * struct xilinx_dpdma_hw_desc - DPDMA hardware descriptor
179 * struct xilinx_dpdma_sw_desc - DPDMA software descriptor
180 * @hw: DPDMA hardware descriptor
191 * struct xilinx_dpdma_tx_desc - DPDMA transaction descriptor
208 * struct xilinx_dpdma_chan - DPDMA channel
215 * @video_group: flag if multi-channel operation is needed for video channels
222 * @xdev: DPDMA device
250 * struct xilinx_dpdma_device - DPDMA device
256 * @chan: DPDMA channels
271 /* -----------------------------------------------------------------------------
302 if (chan->id == dpdma_debugfs.chan_id) in xilinx_dpdma_debugfs_desc_done_irq()
329 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
333 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
340 return -EINVAL; in xilinx_dpdma_debugfs_desc_done_irq_write()
367 return -EINVAL; in xilinx_dpdma_debugfs_read()
372 return -ENOMEM; in xilinx_dpdma_debugfs_read()
387 ret = -EFAULT; in xilinx_dpdma_debugfs_read()
408 return -EINVAL; in xilinx_dpdma_debugfs_write()
412 return -EBUSY; in xilinx_dpdma_debugfs_write()
416 return -ENOMEM; in xilinx_dpdma_debugfs_write()
432 ret = -EINVAL; in xilinx_dpdma_debugfs_write()
459 dent = debugfs_create_file("testcase", 0444, xdev->common.dbg_dev_root, in xilinx_dpdma_debugfs_init()
462 dev_err(xdev->dev, "Failed to create debugfs testcase file\n"); in xilinx_dpdma_debugfs_init()
475 /* -----------------------------------------------------------------------------
499 /* -----------------------------------------------------------------------------
504 * xilinx_dpdma_sw_desc_set_dma_addrs - Set DMA addresses in the descriptor
505 * @xdev: DPDMA device
522 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_sw_desc_set_dma_addrs()
525 hw_desc->src_addr = lower_32_bits(dma_addr[0]); in xilinx_dpdma_sw_desc_set_dma_addrs()
526 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
527 hw_desc->addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
532 u32 *addr = &hw_desc->src_addr2; in xilinx_dpdma_sw_desc_set_dma_addrs()
534 addr[i - 1] = lower_32_bits(dma_addr[i]); in xilinx_dpdma_sw_desc_set_dma_addrs()
536 if (xdev->ext_addr) { in xilinx_dpdma_sw_desc_set_dma_addrs()
537 u32 *addr_ext = &hw_desc->addr_ext_23; in xilinx_dpdma_sw_desc_set_dma_addrs()
541 addr_msb <<= 16 * ((i - 1) % 2); in xilinx_dpdma_sw_desc_set_dma_addrs()
542 addr_ext[(i - 1) / 2] |= addr_msb; in xilinx_dpdma_sw_desc_set_dma_addrs()
549 prev->hw.next_desc = lower_32_bits(sw_desc->dma_addr); in xilinx_dpdma_sw_desc_set_dma_addrs()
550 if (xdev->ext_addr) in xilinx_dpdma_sw_desc_set_dma_addrs()
551 prev->hw.addr_ext |= in xilinx_dpdma_sw_desc_set_dma_addrs()
553 upper_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_sw_desc_set_dma_addrs()
557 * xilinx_dpdma_chan_alloc_sw_desc - Allocate a software descriptor
558 * @chan: DPDMA channel
570 sw_desc = dma_pool_zalloc(chan->desc_pool, GFP_ATOMIC, &dma_addr); in xilinx_dpdma_chan_alloc_sw_desc()
574 sw_desc->dma_addr = dma_addr; in xilinx_dpdma_chan_alloc_sw_desc()
580 * xilinx_dpdma_chan_free_sw_desc - Free a software descriptor
581 * @chan: DPDMA channel
590 dma_pool_free(chan->desc_pool, sw_desc, sw_desc->dma_addr); in xilinx_dpdma_chan_free_sw_desc()
594 * xilinx_dpdma_chan_dump_tx_desc - Dump a tx descriptor
595 * @chan: DPDMA channel
604 struct device *dev = chan->xdev->dev; in xilinx_dpdma_chan_dump_tx_desc()
607 dev_dbg(dev, "------- TX descriptor dump start -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
608 dev_dbg(dev, "------- channel ID = %d -------\n", chan->id); in xilinx_dpdma_chan_dump_tx_desc()
610 list_for_each_entry(sw_desc, &tx_desc->descriptors, node) { in xilinx_dpdma_chan_dump_tx_desc()
611 struct xilinx_dpdma_hw_desc *hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_dump_tx_desc()
613 dev_dbg(dev, "------- HW descriptor %d -------\n", i++); in xilinx_dpdma_chan_dump_tx_desc()
614 dev_dbg(dev, "descriptor DMA addr: %pad\n", &sw_desc->dma_addr); in xilinx_dpdma_chan_dump_tx_desc()
615 dev_dbg(dev, "control: 0x%08x\n", hw_desc->control); in xilinx_dpdma_chan_dump_tx_desc()
616 dev_dbg(dev, "desc_id: 0x%08x\n", hw_desc->desc_id); in xilinx_dpdma_chan_dump_tx_desc()
617 dev_dbg(dev, "xfer_size: 0x%08x\n", hw_desc->xfer_size); in xilinx_dpdma_chan_dump_tx_desc()
618 dev_dbg(dev, "hsize_stride: 0x%08x\n", hw_desc->hsize_stride); in xilinx_dpdma_chan_dump_tx_desc()
619 dev_dbg(dev, "timestamp_lsb: 0x%08x\n", hw_desc->timestamp_lsb); in xilinx_dpdma_chan_dump_tx_desc()
620 dev_dbg(dev, "timestamp_msb: 0x%08x\n", hw_desc->timestamp_msb); in xilinx_dpdma_chan_dump_tx_desc()
621 dev_dbg(dev, "addr_ext: 0x%08x\n", hw_desc->addr_ext); in xilinx_dpdma_chan_dump_tx_desc()
622 dev_dbg(dev, "next_desc: 0x%08x\n", hw_desc->next_desc); in xilinx_dpdma_chan_dump_tx_desc()
623 dev_dbg(dev, "src_addr: 0x%08x\n", hw_desc->src_addr); in xilinx_dpdma_chan_dump_tx_desc()
624 dev_dbg(dev, "addr_ext_23: 0x%08x\n", hw_desc->addr_ext_23); in xilinx_dpdma_chan_dump_tx_desc()
625 dev_dbg(dev, "addr_ext_45: 0x%08x\n", hw_desc->addr_ext_45); in xilinx_dpdma_chan_dump_tx_desc()
626 dev_dbg(dev, "src_addr2: 0x%08x\n", hw_desc->src_addr2); in xilinx_dpdma_chan_dump_tx_desc()
627 dev_dbg(dev, "src_addr3: 0x%08x\n", hw_desc->src_addr3); in xilinx_dpdma_chan_dump_tx_desc()
628 dev_dbg(dev, "src_addr4: 0x%08x\n", hw_desc->src_addr4); in xilinx_dpdma_chan_dump_tx_desc()
629 dev_dbg(dev, "src_addr5: 0x%08x\n", hw_desc->src_addr5); in xilinx_dpdma_chan_dump_tx_desc()
630 dev_dbg(dev, "crc: 0x%08x\n", hw_desc->crc); in xilinx_dpdma_chan_dump_tx_desc()
633 dev_dbg(dev, "------- TX descriptor dump end -------\n"); in xilinx_dpdma_chan_dump_tx_desc()
637 * xilinx_dpdma_chan_alloc_tx_desc - Allocate a transaction descriptor
638 * @chan: DPDMA channel
653 INIT_LIST_HEAD(&tx_desc->descriptors); in xilinx_dpdma_chan_alloc_tx_desc()
654 tx_desc->chan = chan; in xilinx_dpdma_chan_alloc_tx_desc()
655 tx_desc->error = false; in xilinx_dpdma_chan_alloc_tx_desc()
661 * xilinx_dpdma_chan_free_tx_desc - Free a virtual DMA descriptor
676 list_for_each_entry_safe(sw_desc, next, &desc->descriptors, node) { in xilinx_dpdma_chan_free_tx_desc()
677 list_del(&sw_desc->node); in xilinx_dpdma_chan_free_tx_desc()
678 xilinx_dpdma_chan_free_sw_desc(desc->chan, sw_desc); in xilinx_dpdma_chan_free_tx_desc()
685 * xilinx_dpdma_chan_prep_interleaved_dma - Prepare an interleaved dma
687 * @chan: DPDMA channel
693 * Return: A DPDMA TX descriptor on success, or NULL.
702 size_t hsize = xt->sgl[0].size; in xilinx_dpdma_chan_prep_interleaved_dma()
703 size_t stride = hsize + xt->sgl[0].icg; in xilinx_dpdma_chan_prep_interleaved_dma()
705 if (!IS_ALIGNED(xt->src_start, XILINX_DPDMA_ALIGN_BYTES)) { in xilinx_dpdma_chan_prep_interleaved_dma()
706 dev_err(chan->xdev->dev, in xilinx_dpdma_chan_prep_interleaved_dma()
708 chan->id, XILINX_DPDMA_ALIGN_BYTES); in xilinx_dpdma_chan_prep_interleaved_dma()
718 xilinx_dpdma_chan_free_tx_desc(&tx_desc->vdesc); in xilinx_dpdma_chan_prep_interleaved_dma()
722 xilinx_dpdma_sw_desc_set_dma_addrs(chan->xdev, sw_desc, sw_desc, in xilinx_dpdma_chan_prep_interleaved_dma()
723 &xt->src_start, 1); in xilinx_dpdma_chan_prep_interleaved_dma()
725 hw_desc = &sw_desc->hw; in xilinx_dpdma_chan_prep_interleaved_dma()
727 hw_desc->xfer_size = hsize * xt->numf; in xilinx_dpdma_chan_prep_interleaved_dma()
728 hw_desc->hsize_stride = in xilinx_dpdma_chan_prep_interleaved_dma()
732 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_PREEMBLE; in xilinx_dpdma_chan_prep_interleaved_dma()
733 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_COMPLETE_INTR; in xilinx_dpdma_chan_prep_interleaved_dma()
734 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_IGNORE_DONE; in xilinx_dpdma_chan_prep_interleaved_dma()
735 hw_desc->control |= XILINX_DPDMA_DESC_CONTROL_LAST_OF_FRAME; in xilinx_dpdma_chan_prep_interleaved_dma()
737 list_add_tail(&sw_desc->node, &tx_desc->descriptors); in xilinx_dpdma_chan_prep_interleaved_dma()
742 /* -----------------------------------------------------------------------------
743 * DPDMA Channel Operations
747 * xilinx_dpdma_chan_enable - Enable the channel
748 * @chan: DPDMA channel
756 reg = (XILINX_DPDMA_INTR_CHAN_MASK << chan->id) in xilinx_dpdma_chan_enable()
758 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_enable()
759 reg = (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id) in xilinx_dpdma_chan_enable()
761 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_enable()
770 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, reg); in xilinx_dpdma_chan_enable()
774 * xilinx_dpdma_chan_disable - Disable the channel
775 * @chan: DPDMA channel
783 reg = XILINX_DPDMA_INTR_CHAN_MASK << chan->id; in xilinx_dpdma_chan_disable()
784 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, reg); in xilinx_dpdma_chan_disable()
785 reg = XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id; in xilinx_dpdma_chan_disable()
786 dpdma_write(chan->xdev->reg, XILINX_DPDMA_EIEN, reg); in xilinx_dpdma_chan_disable()
788 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_ENABLE); in xilinx_dpdma_chan_disable()
792 * xilinx_dpdma_chan_pause - Pause the channel
793 * @chan: DPDMA channel
799 dpdma_set(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_pause()
803 * xilinx_dpdma_chan_unpause - Unpause the channel
804 * @chan: DPDMA channel
810 dpdma_clr(chan->reg, XILINX_DPDMA_CH_CNTL, XILINX_DPDMA_CH_CNTL_PAUSE); in xilinx_dpdma_chan_unpause()
815 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_video_group_ready()
820 if (xdev->chan[i]->video_group && !xdev->chan[i]->running) in xilinx_dpdma_chan_video_group_ready()
823 if (xdev->chan[i]->video_group) in xilinx_dpdma_chan_video_group_ready()
831 * xilinx_dpdma_chan_queue_transfer - Queue the next transfer
832 * @chan: DPDMA channel
839 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_queue_transfer()
846 lockdep_assert_held(&chan->lock); in xilinx_dpdma_chan_queue_transfer()
848 if (chan->desc.pending) in xilinx_dpdma_chan_queue_transfer()
851 if (!chan->running) { in xilinx_dpdma_chan_queue_transfer()
854 chan->first_frame = true; in xilinx_dpdma_chan_queue_transfer()
855 chan->running = true; in xilinx_dpdma_chan_queue_transfer()
858 vdesc = vchan_next_desc(&chan->vchan); in xilinx_dpdma_chan_queue_transfer()
863 chan->desc.pending = desc; in xilinx_dpdma_chan_queue_transfer()
864 list_del(&desc->vdesc.node); in xilinx_dpdma_chan_queue_transfer()
870 list_for_each_entry(sw_desc, &desc->descriptors, node) in xilinx_dpdma_chan_queue_transfer()
871 sw_desc->hw.desc_id = desc->vdesc.tx.cookie in xilinx_dpdma_chan_queue_transfer()
874 sw_desc = list_first_entry(&desc->descriptors, in xilinx_dpdma_chan_queue_transfer()
876 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR, in xilinx_dpdma_chan_queue_transfer()
877 lower_32_bits(sw_desc->dma_addr)); in xilinx_dpdma_chan_queue_transfer()
878 if (xdev->ext_addr) in xilinx_dpdma_chan_queue_transfer()
879 dpdma_write(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE, in xilinx_dpdma_chan_queue_transfer()
881 upper_32_bits(sw_desc->dma_addr))); in xilinx_dpdma_chan_queue_transfer()
883 first_frame = chan->first_frame; in xilinx_dpdma_chan_queue_transfer()
884 chan->first_frame = false; in xilinx_dpdma_chan_queue_transfer()
886 if (chan->video_group) { in xilinx_dpdma_chan_queue_transfer()
895 channels = BIT(chan->id); in xilinx_dpdma_chan_queue_transfer()
903 dpdma_write(xdev->reg, XILINX_DPDMA_GBL, reg); in xilinx_dpdma_chan_queue_transfer()
907 * xilinx_dpdma_chan_ostand - Number of outstanding transactions
908 * @chan: DPDMA channel
917 dpdma_read(chan->reg, XILINX_DPDMA_CH_STATUS)); in xilinx_dpdma_chan_ostand()
921 * xilinx_dpdma_chan_notify_no_ostand - Notify no outstanding transaction event
922 * @chan: DPDMA channel
927 * should be re-enabled when this event is handled. If the channel status
931 * Return: 0 on success. On failure, -EWOULDBLOCK if there's still outstanding
940 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_notify_no_ostand()
942 chan->id, cnt); in xilinx_dpdma_chan_notify_no_ostand()
943 return -EWOULDBLOCK; in xilinx_dpdma_chan_notify_no_ostand()
947 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_chan_notify_no_ostand()
948 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_notify_no_ostand()
949 wake_up(&chan->wait_to_stop); in xilinx_dpdma_chan_notify_no_ostand()
955 * xilinx_dpdma_chan_wait_no_ostand - Wait for the no outstanding irq
956 * @chan: DPDMA channel
961 * Return: 0 on success. On failure, -ETIMEOUT for time out, or the error code
969 ret = wait_event_interruptible_timeout(chan->wait_to_stop, in xilinx_dpdma_chan_wait_no_ostand()
973 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_wait_no_ostand()
974 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_wait_no_ostand()
978 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_wait_no_ostand()
979 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_wait_no_ostand()
982 return -ETIMEDOUT; in xilinx_dpdma_chan_wait_no_ostand()
988 * xilinx_dpdma_chan_poll_no_ostand - Poll the outstanding transaction status
989 * @chan: DPDMA channel
995 * Return: 0 on success, or -ETIMEDOUT.
1005 } while (loop-- > 0 && cnt); in xilinx_dpdma_chan_poll_no_ostand()
1008 dpdma_write(chan->xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_poll_no_ostand()
1009 XILINX_DPDMA_INTR_NO_OSTAND(chan->id)); in xilinx_dpdma_chan_poll_no_ostand()
1013 dev_err(chan->xdev->dev, "chan%u: not ready to stop: %d trans\n", in xilinx_dpdma_chan_poll_no_ostand()
1014 chan->id, xilinx_dpdma_chan_ostand(chan)); in xilinx_dpdma_chan_poll_no_ostand()
1016 return -ETIMEDOUT; in xilinx_dpdma_chan_poll_no_ostand()
1020 * xilinx_dpdma_chan_stop - Stop the channel
1021 * @chan: DPDMA channel
1026 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1037 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1039 chan->running = false; in xilinx_dpdma_chan_stop()
1040 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_stop()
1046 * xilinx_dpdma_chan_done_irq - Handle hardware descriptor completion
1047 * @chan: DPDMA channel
1049 * Handle completion of the currently active descriptor (@chan->desc.active). As
1059 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1063 active = chan->desc.active; in xilinx_dpdma_chan_done_irq()
1065 vchan_cyclic_callback(&active->vdesc); in xilinx_dpdma_chan_done_irq()
1067 dev_warn(chan->xdev->dev, in xilinx_dpdma_chan_done_irq()
1069 chan->id); in xilinx_dpdma_chan_done_irq()
1071 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_done_irq()
1075 * xilinx_dpdma_chan_vsync_irq - Handle hardware descriptor scheduling
1076 * @chan: DPDMA channel
1089 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1091 pending = chan->desc.pending; in xilinx_dpdma_chan_vsync_irq()
1092 if (!chan->running || !pending) in xilinx_dpdma_chan_vsync_irq()
1095 desc_id = dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_ID) in xilinx_dpdma_chan_vsync_irq()
1099 sw_desc = list_first_entry(&pending->descriptors, in xilinx_dpdma_chan_vsync_irq()
1101 if (sw_desc->hw.desc_id != desc_id) { in xilinx_dpdma_chan_vsync_irq()
1102 dev_dbg(chan->xdev->dev, in xilinx_dpdma_chan_vsync_irq()
1104 chan->id, sw_desc->hw.desc_id, desc_id); in xilinx_dpdma_chan_vsync_irq()
1112 if (chan->desc.active) in xilinx_dpdma_chan_vsync_irq()
1113 vchan_cookie_complete(&chan->desc.active->vdesc); in xilinx_dpdma_chan_vsync_irq()
1114 chan->desc.active = pending; in xilinx_dpdma_chan_vsync_irq()
1115 chan->desc.pending = NULL; in xilinx_dpdma_chan_vsync_irq()
1120 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_vsync_irq()
1124 * xilinx_dpdma_chan_err - Detect any channel error
1125 * @chan: DPDMA channel
1137 if (chan->running && in xilinx_dpdma_chan_err()
1138 ((isr & (XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id)) || in xilinx_dpdma_chan_err()
1139 (eisr & (XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id)))) in xilinx_dpdma_chan_err()
1146 * xilinx_dpdma_chan_handle_err - DPDMA channel error handling
1147 * @chan: DPDMA channel
1156 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_handle_err()
1160 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1162 dev_dbg(xdev->dev, "chan%u: cur desc addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1163 chan->id, in xilinx_dpdma_chan_handle_err()
1164 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDRE), in xilinx_dpdma_chan_handle_err()
1165 dpdma_read(chan->reg, XILINX_DPDMA_CH_DESC_START_ADDR)); in xilinx_dpdma_chan_handle_err()
1166 dev_dbg(xdev->dev, "chan%u: cur payload addr = 0x%04x%08x\n", in xilinx_dpdma_chan_handle_err()
1167 chan->id, in xilinx_dpdma_chan_handle_err()
1168 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDRE), in xilinx_dpdma_chan_handle_err()
1169 dpdma_read(chan->reg, XILINX_DPDMA_CH_PYLD_CUR_ADDR)); in xilinx_dpdma_chan_handle_err()
1172 chan->running = false; in xilinx_dpdma_chan_handle_err()
1174 if (!chan->desc.active) in xilinx_dpdma_chan_handle_err()
1177 active = chan->desc.active; in xilinx_dpdma_chan_handle_err()
1178 chan->desc.active = NULL; in xilinx_dpdma_chan_handle_err()
1182 if (active->error) in xilinx_dpdma_chan_handle_err()
1183 dev_dbg(xdev->dev, "chan%u: repeated error on desc\n", in xilinx_dpdma_chan_handle_err()
1184 chan->id); in xilinx_dpdma_chan_handle_err()
1187 if (!chan->desc.pending && in xilinx_dpdma_chan_handle_err()
1188 list_empty(&chan->vchan.desc_issued)) { in xilinx_dpdma_chan_handle_err()
1189 active->error = true; in xilinx_dpdma_chan_handle_err()
1190 list_add_tail(&active->vdesc.node, in xilinx_dpdma_chan_handle_err()
1191 &chan->vchan.desc_issued); in xilinx_dpdma_chan_handle_err()
1193 xilinx_dpdma_chan_free_tx_desc(&active->vdesc); in xilinx_dpdma_chan_handle_err()
1197 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_handle_err()
1200 /* -----------------------------------------------------------------------------
1212 if (xt->dir != DMA_MEM_TO_DEV) in xilinx_dpdma_prep_interleaved_dma()
1215 if (!xt->numf || !xt->sgl[0].size) in xilinx_dpdma_prep_interleaved_dma()
1225 vchan_tx_prep(&chan->vchan, &desc->vdesc, flags | DMA_CTRL_ACK); in xilinx_dpdma_prep_interleaved_dma()
1227 return &desc->vdesc.tx; in xilinx_dpdma_prep_interleaved_dma()
1231 * xilinx_dpdma_alloc_chan_resources - Allocate resources for the channel
1236 * Return: 0 on success, or -ENOMEM if failed to allocate a pool.
1243 chan->desc_pool = dma_pool_create(dev_name(chan->xdev->dev), in xilinx_dpdma_alloc_chan_resources()
1244 chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1247 if (!chan->desc_pool) { in xilinx_dpdma_alloc_chan_resources()
1248 dev_err(chan->xdev->dev, in xilinx_dpdma_alloc_chan_resources()
1250 chan->id); in xilinx_dpdma_alloc_chan_resources()
1251 return -ENOMEM; in xilinx_dpdma_alloc_chan_resources()
1258 * xilinx_dpdma_free_chan_resources - Free all resources for the channel
1268 vchan_free_chan_resources(&chan->vchan); in xilinx_dpdma_free_chan_resources()
1270 dma_pool_destroy(chan->desc_pool); in xilinx_dpdma_free_chan_resources()
1271 chan->desc_pool = NULL; in xilinx_dpdma_free_chan_resources()
1279 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1280 if (vchan_issue_pending(&chan->vchan)) in xilinx_dpdma_issue_pending()
1282 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_issue_pending()
1292 * The destination address doesn't need to be specified as the DPDMA is in xilinx_dpdma_config()
1295 * fixed both on the DPDMA side and on the DP controller side. in xilinx_dpdma_config()
1298 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_config()
1304 if (chan->id <= ZYNQMP_DPDMA_VIDEO2) in xilinx_dpdma_config()
1305 chan->video_group = config->slave_id != 0; in xilinx_dpdma_config()
1307 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_config()
1327 * xilinx_dpdma_terminate_all - Terminate the channel and descriptors
1339 * Return: 0 on success, or -ETIMEDOUT if the channel failed to stop.
1344 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_terminate_all()
1350 if (chan->video_group) { in xilinx_dpdma_terminate_all()
1352 if (xdev->chan[i]->video_group && in xilinx_dpdma_terminate_all()
1353 xdev->chan[i]->running) { in xilinx_dpdma_terminate_all()
1354 xilinx_dpdma_chan_pause(xdev->chan[i]); in xilinx_dpdma_terminate_all()
1355 xdev->chan[i]->video_group = false; in xilinx_dpdma_terminate_all()
1363 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1364 vchan_get_all_descriptors(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1365 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_terminate_all()
1367 vchan_dma_desc_free_list(&chan->vchan, &descriptors); in xilinx_dpdma_terminate_all()
1373 * xilinx_dpdma_synchronize - Synchronize callback execution
1392 spin_lock_irqsave(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1393 if (chan->desc.pending) { in xilinx_dpdma_synchronize()
1394 vchan_terminate_vdesc(&chan->desc.pending->vdesc); in xilinx_dpdma_synchronize()
1395 chan->desc.pending = NULL; in xilinx_dpdma_synchronize()
1397 if (chan->desc.active) { in xilinx_dpdma_synchronize()
1398 vchan_terminate_vdesc(&chan->desc.active->vdesc); in xilinx_dpdma_synchronize()
1399 chan->desc.active = NULL; in xilinx_dpdma_synchronize()
1401 spin_unlock_irqrestore(&chan->vchan.lock, flags); in xilinx_dpdma_synchronize()
1403 vchan_synchronize(&chan->vchan); in xilinx_dpdma_synchronize()
1406 /* -----------------------------------------------------------------------------
1411 * xilinx_dpdma_err - Detect any global error
1427 * xilinx_dpdma_handle_err_irq - Handle DPDMA error interrupt
1428 * @xdev: DPDMA device
1433 * corresponding error interrupts, and those should be re-enabled once handling
1442 dev_dbg_ratelimited(xdev->dev, in xilinx_dpdma_handle_err_irq()
1447 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, in xilinx_dpdma_handle_err_irq()
1449 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, in xilinx_dpdma_handle_err_irq()
1452 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_handle_err_irq()
1453 if (err || xilinx_dpdma_chan_err(xdev->chan[i], isr, eisr)) in xilinx_dpdma_handle_err_irq()
1454 tasklet_schedule(&xdev->chan[i]->err_task); in xilinx_dpdma_handle_err_irq()
1458 * xilinx_dpdma_enable_irq - Enable interrupts
1459 * @xdev: DPDMA device
1465 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_enable_irq()
1466 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_enable_irq()
1470 * xilinx_dpdma_disable_irq - Disable interrupts
1471 * @xdev: DPDMA device
1477 dpdma_write(xdev->reg, XILINX_DPDMA_IDS, XILINX_DPDMA_INTR_ALL); in xilinx_dpdma_disable_irq()
1478 dpdma_write(xdev->reg, XILINX_DPDMA_EIDS, XILINX_DPDMA_EINTR_ALL); in xilinx_dpdma_disable_irq()
1482 * xilinx_dpdma_chan_err_task - Per channel tasklet for error handling
1487 * re-enable channel error interrupts, and restart the channel if needed.
1492 struct xilinx_dpdma_device *xdev = chan->xdev; in xilinx_dpdma_chan_err_task()
1500 dpdma_write(xdev->reg, XILINX_DPDMA_IEN, in xilinx_dpdma_chan_err_task()
1501 XILINX_DPDMA_INTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1502 dpdma_write(xdev->reg, XILINX_DPDMA_EIEN, in xilinx_dpdma_chan_err_task()
1503 XILINX_DPDMA_EINTR_CHAN_ERR_MASK << chan->id); in xilinx_dpdma_chan_err_task()
1505 spin_lock_irqsave(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1507 spin_unlock_irqrestore(&chan->lock, flags); in xilinx_dpdma_chan_err_task()
1518 status = dpdma_read(xdev->reg, XILINX_DPDMA_ISR); in xilinx_dpdma_irq_handler()
1519 error = dpdma_read(xdev->reg, XILINX_DPDMA_EISR); in xilinx_dpdma_irq_handler()
1523 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, status); in xilinx_dpdma_irq_handler()
1524 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, error); in xilinx_dpdma_irq_handler()
1531 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in xilinx_dpdma_irq_handler()
1532 struct xilinx_dpdma_chan *chan = xdev->chan[i]; in xilinx_dpdma_irq_handler()
1541 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1542 xilinx_dpdma_chan_done_irq(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1547 for_each_set_bit(i, &mask, ARRAY_SIZE(xdev->chan)) in xilinx_dpdma_irq_handler()
1548 xilinx_dpdma_chan_notify_no_ostand(xdev->chan[i]); in xilinx_dpdma_irq_handler()
1558 /* -----------------------------------------------------------------------------
1567 chan = devm_kzalloc(xdev->dev, sizeof(*chan), GFP_KERNEL); in xilinx_dpdma_chan_init()
1569 return -ENOMEM; in xilinx_dpdma_chan_init()
1571 chan->id = chan_id; in xilinx_dpdma_chan_init()
1572 chan->reg = xdev->reg + XILINX_DPDMA_CH_BASE in xilinx_dpdma_chan_init()
1573 + XILINX_DPDMA_CH_OFFSET * chan->id; in xilinx_dpdma_chan_init()
1574 chan->running = false; in xilinx_dpdma_chan_init()
1575 chan->xdev = xdev; in xilinx_dpdma_chan_init()
1577 spin_lock_init(&chan->lock); in xilinx_dpdma_chan_init()
1578 init_waitqueue_head(&chan->wait_to_stop); in xilinx_dpdma_chan_init()
1580 tasklet_setup(&chan->err_task, xilinx_dpdma_chan_err_task); in xilinx_dpdma_chan_init()
1582 chan->vchan.desc_free = xilinx_dpdma_chan_free_tx_desc; in xilinx_dpdma_chan_init()
1583 vchan_init(&chan->vchan, &xdev->common); in xilinx_dpdma_chan_init()
1585 xdev->chan[chan->id] = chan; in xilinx_dpdma_chan_init()
1595 tasklet_kill(&chan->err_task); in xilinx_dpdma_chan_remove()
1596 list_del(&chan->vchan.chan.device_node); in xilinx_dpdma_chan_remove()
1602 struct xilinx_dpdma_device *xdev = ofdma->of_dma_data; in of_dma_xilinx_xlate()
1603 u32 chan_id = dma_spec->args[0]; in of_dma_xilinx_xlate()
1605 if (chan_id >= ARRAY_SIZE(xdev->chan)) in of_dma_xilinx_xlate()
1608 if (!xdev->chan[chan_id]) in of_dma_xilinx_xlate()
1611 return dma_get_slave_channel(&xdev->chan[chan_id]->vchan.chan); in of_dma_xilinx_xlate()
1623 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) { in dpdma_hw_init()
1624 reg = xdev->reg + XILINX_DPDMA_CH_BASE in dpdma_hw_init()
1630 dpdma_write(xdev->reg, XILINX_DPDMA_ISR, XILINX_DPDMA_INTR_ALL); in dpdma_hw_init()
1631 dpdma_write(xdev->reg, XILINX_DPDMA_EISR, XILINX_DPDMA_EINTR_ALL); in dpdma_hw_init()
1641 xdev = devm_kzalloc(&pdev->dev, sizeof(*xdev), GFP_KERNEL); in xilinx_dpdma_probe()
1643 return -ENOMEM; in xilinx_dpdma_probe()
1645 xdev->dev = &pdev->dev; in xilinx_dpdma_probe()
1646 xdev->ext_addr = sizeof(dma_addr_t) > 4; in xilinx_dpdma_probe()
1648 INIT_LIST_HEAD(&xdev->common.channels); in xilinx_dpdma_probe()
1652 xdev->axi_clk = devm_clk_get(xdev->dev, "axi_clk"); in xilinx_dpdma_probe()
1653 if (IS_ERR(xdev->axi_clk)) in xilinx_dpdma_probe()
1654 return PTR_ERR(xdev->axi_clk); in xilinx_dpdma_probe()
1656 xdev->reg = devm_platform_ioremap_resource(pdev, 0); in xilinx_dpdma_probe()
1657 if (IS_ERR(xdev->reg)) in xilinx_dpdma_probe()
1658 return PTR_ERR(xdev->reg); in xilinx_dpdma_probe()
1662 xdev->irq = platform_get_irq(pdev, 0); in xilinx_dpdma_probe()
1663 if (xdev->irq < 0) { in xilinx_dpdma_probe()
1664 dev_err(xdev->dev, "failed to get platform irq\n"); in xilinx_dpdma_probe()
1665 return xdev->irq; in xilinx_dpdma_probe()
1668 ret = request_irq(xdev->irq, xilinx_dpdma_irq_handler, IRQF_SHARED, in xilinx_dpdma_probe()
1669 dev_name(xdev->dev), xdev); in xilinx_dpdma_probe()
1671 dev_err(xdev->dev, "failed to request IRQ\n"); in xilinx_dpdma_probe()
1675 ddev = &xdev->common; in xilinx_dpdma_probe()
1676 ddev->dev = &pdev->dev; in xilinx_dpdma_probe()
1678 dma_cap_set(DMA_SLAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1679 dma_cap_set(DMA_PRIVATE, ddev->cap_mask); in xilinx_dpdma_probe()
1680 dma_cap_set(DMA_INTERLEAVE, ddev->cap_mask); in xilinx_dpdma_probe()
1681 dma_cap_set(DMA_REPEAT, ddev->cap_mask); in xilinx_dpdma_probe()
1682 dma_cap_set(DMA_LOAD_EOT, ddev->cap_mask); in xilinx_dpdma_probe()
1683 ddev->copy_align = fls(XILINX_DPDMA_ALIGN_BYTES - 1); in xilinx_dpdma_probe()
1685 ddev->device_alloc_chan_resources = xilinx_dpdma_alloc_chan_resources; in xilinx_dpdma_probe()
1686 ddev->device_free_chan_resources = xilinx_dpdma_free_chan_resources; in xilinx_dpdma_probe()
1687 ddev->device_prep_interleaved_dma = xilinx_dpdma_prep_interleaved_dma; in xilinx_dpdma_probe()
1689 ddev->device_tx_status = dma_cookie_status; in xilinx_dpdma_probe()
1690 ddev->device_issue_pending = xilinx_dpdma_issue_pending; in xilinx_dpdma_probe()
1691 ddev->device_config = xilinx_dpdma_config; in xilinx_dpdma_probe()
1692 ddev->device_pause = xilinx_dpdma_pause; in xilinx_dpdma_probe()
1693 ddev->device_resume = xilinx_dpdma_resume; in xilinx_dpdma_probe()
1694 ddev->device_terminate_all = xilinx_dpdma_terminate_all; in xilinx_dpdma_probe()
1695 ddev->device_synchronize = xilinx_dpdma_synchronize; in xilinx_dpdma_probe()
1696 ddev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED); in xilinx_dpdma_probe()
1697 ddev->directions = BIT(DMA_MEM_TO_DEV); in xilinx_dpdma_probe()
1698 ddev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR; in xilinx_dpdma_probe()
1700 for (i = 0; i < ARRAY_SIZE(xdev->chan); ++i) { in xilinx_dpdma_probe()
1703 dev_err(xdev->dev, "failed to initialize channel %u\n", in xilinx_dpdma_probe()
1709 ret = clk_prepare_enable(xdev->axi_clk); in xilinx_dpdma_probe()
1711 dev_err(xdev->dev, "failed to enable the axi clock\n"); in xilinx_dpdma_probe()
1717 dev_err(xdev->dev, "failed to register the dma device\n"); in xilinx_dpdma_probe()
1721 ret = of_dma_controller_register(xdev->dev->of_node, in xilinx_dpdma_probe()
1724 dev_err(xdev->dev, "failed to register DMA to DT DMA helper\n"); in xilinx_dpdma_probe()
1732 dev_info(&pdev->dev, "Xilinx DPDMA engine is probed\n"); in xilinx_dpdma_probe()
1739 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_probe()
1741 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_probe()
1742 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_probe()
1744 free_irq(xdev->irq, xdev); in xilinx_dpdma_probe()
1755 free_irq(xdev->irq, xdev); in xilinx_dpdma_remove()
1758 of_dma_controller_free(pdev->dev.of_node); in xilinx_dpdma_remove()
1759 dma_async_device_unregister(&xdev->common); in xilinx_dpdma_remove()
1760 clk_disable_unprepare(xdev->axi_clk); in xilinx_dpdma_remove()
1762 for (i = 0; i < ARRAY_SIZE(xdev->chan); i++) in xilinx_dpdma_remove()
1763 xilinx_dpdma_chan_remove(xdev->chan[i]); in xilinx_dpdma_remove()
1769 { .compatible = "xlnx,zynqmp-dpdma",},
1778 .name = "xilinx-zynqmp-dpdma",
1786 MODULE_DESCRIPTION("Xilinx ZynqMP DPDMA driver");