Lines Matching +full:jz4780 +full:- +full:dma
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Ingenic JZ4780 DMA controller
6 * Author: Alex Smith <alex@alex-smith.me.uk>
21 #include "virt-dma.h"
37 /* Per-channel registers. */
98 * struct jz4780_dma_hwdesc - descriptor structure read by the DMA controller.
178 return container_of(jzchan->vchan.chan.device, struct jz4780_dma_dev, in jz4780_dma_chan_parent()
185 return readl(jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); in jz4780_dma_chn_readl()
191 writel(val, jzdma->chn_base + reg + JZ_DMA_REG_CHAN(chn)); in jz4780_dma_chn_writel()
197 return readl(jzdma->ctrl_base + reg); in jz4780_dma_ctrl_readl()
203 writel(val, jzdma->ctrl_base + reg); in jz4780_dma_ctrl_writel()
209 if (jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) { in jz4780_dma_chan_enable()
212 if (jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC) in jz4780_dma_chan_enable()
224 if ((jzdma->soc_data->flags & JZ_SOC_DATA_PER_CHAN_PM) && in jz4780_dma_chan_disable()
225 !(jzdma->soc_data->flags & JZ_SOC_DATA_NO_DCKES_DCKEC)) in jz4780_dma_chan_disable()
242 desc->desc = dma_pool_alloc(jzchan->desc_pool, GFP_NOWAIT, in jz4780_dma_desc_alloc()
243 &desc->desc_phys); in jz4780_dma_desc_alloc()
244 if (!desc->desc) { in jz4780_dma_desc_alloc()
249 desc->count = count; in jz4780_dma_desc_alloc()
250 desc->type = type; in jz4780_dma_desc_alloc()
257 struct jz4780_dma_chan *jzchan = to_jz4780_dma_chan(vdesc->tx.chan); in jz4780_dma_desc_free()
259 dma_pool_free(jzchan->desc_pool, desc->desc, desc->desc_phys); in jz4780_dma_desc_free()
267 int ord = ffs(val) - 1; in jz4780_dma_transfer_size()
278 else if (ord > jzdma->soc_data->transfer_ord_max) in jz4780_dma_transfer_size()
279 ord = jzdma->soc_data->transfer_ord_max; in jz4780_dma_transfer_size()
305 struct dma_slave_config *config = &jzchan->config; in jz4780_dma_setup_hwdesc()
309 desc->dcm = JZ_DMA_DCM_SAI; in jz4780_dma_setup_hwdesc()
310 desc->dsa = addr; in jz4780_dma_setup_hwdesc()
311 desc->dta = config->dst_addr; in jz4780_dma_setup_hwdesc()
313 width = config->dst_addr_width; in jz4780_dma_setup_hwdesc()
314 maxburst = config->dst_maxburst; in jz4780_dma_setup_hwdesc()
316 desc->dcm = JZ_DMA_DCM_DAI; in jz4780_dma_setup_hwdesc()
317 desc->dsa = config->src_addr; in jz4780_dma_setup_hwdesc()
318 desc->dta = addr; in jz4780_dma_setup_hwdesc()
320 width = config->src_addr_width; in jz4780_dma_setup_hwdesc()
321 maxburst = config->src_maxburst; in jz4780_dma_setup_hwdesc()
332 &jzchan->transfer_shift); in jz4780_dma_setup_hwdesc()
342 return -EINVAL; in jz4780_dma_setup_hwdesc()
345 desc->dcm |= tsz << JZ_DMA_DCM_TSZ_SHIFT; in jz4780_dma_setup_hwdesc()
346 desc->dcm |= width << JZ_DMA_DCM_SP_SHIFT; in jz4780_dma_setup_hwdesc()
347 desc->dcm |= width << JZ_DMA_DCM_DP_SHIFT; in jz4780_dma_setup_hwdesc()
349 desc->dtc = len >> jzchan->transfer_shift; in jz4780_dma_setup_hwdesc()
369 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], in jz4780_dma_prep_slave_sg()
374 jz4780_dma_desc_free(&jzchan->desc->vdesc); in jz4780_dma_prep_slave_sg()
378 desc->desc[i].dcm |= JZ_DMA_DCM_TIE; in jz4780_dma_prep_slave_sg()
380 if (i != (sg_len - 1) && in jz4780_dma_prep_slave_sg()
381 !(jzdma->soc_data->flags & JZ_SOC_DATA_BREAK_LINKS)) { in jz4780_dma_prep_slave_sg()
383 desc->desc[i].dcm |= JZ_DMA_DCM_LINK; in jz4780_dma_prep_slave_sg()
390 desc->desc[i].dtc |= in jz4780_dma_prep_slave_sg()
391 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; in jz4780_dma_prep_slave_sg()
395 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); in jz4780_dma_prep_slave_sg()
418 err = jz4780_dma_setup_hwdesc(jzchan, &desc->desc[i], buf_addr, in jz4780_dma_prep_dma_cyclic()
421 jz4780_dma_desc_free(&jzchan->desc->vdesc); in jz4780_dma_prep_dma_cyclic()
433 desc->desc[i].dcm |= JZ_DMA_DCM_TIE | JZ_DMA_DCM_LINK; in jz4780_dma_prep_dma_cyclic()
441 if (i != (periods - 1)) { in jz4780_dma_prep_dma_cyclic()
442 desc->desc[i].dtc |= in jz4780_dma_prep_dma_cyclic()
443 (((i + 1) * sizeof(*desc->desc)) >> 4) << 24; in jz4780_dma_prep_dma_cyclic()
447 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); in jz4780_dma_prep_dma_cyclic()
463 &jzchan->transfer_shift); in jz4780_dma_prep_dma_memcpy()
465 jzchan->transfer_type = JZ_DMA_DRT_AUTO; in jz4780_dma_prep_dma_memcpy()
467 desc->desc[0].dsa = src; in jz4780_dma_prep_dma_memcpy()
468 desc->desc[0].dta = dest; in jz4780_dma_prep_dma_memcpy()
469 desc->desc[0].dcm = JZ_DMA_DCM_TIE | JZ_DMA_DCM_SAI | JZ_DMA_DCM_DAI | in jz4780_dma_prep_dma_memcpy()
473 desc->desc[0].dtc = len >> jzchan->transfer_shift; in jz4780_dma_prep_dma_memcpy()
475 return vchan_tx_prep(&jzchan->vchan, &desc->vdesc, flags); in jz4780_dma_prep_dma_memcpy()
485 if (!jzchan->desc) { in jz4780_dma_begin()
486 vdesc = vchan_next_desc(&jzchan->vchan); in jz4780_dma_begin()
490 list_del(&vdesc->node); in jz4780_dma_begin()
492 jzchan->desc = to_jz4780_dma_desc(vdesc); in jz4780_dma_begin()
493 jzchan->curr_hwdesc = 0; in jz4780_dma_begin()
495 if (jzchan->desc->type == DMA_CYCLIC && vdesc->tx.callback) { in jz4780_dma_begin()
497 * The DMA controller doesn't support triggering an in jz4780_dma_begin()
500 * descriptors. For a cyclic DMA setup the list of in jz4780_dma_begin()
504 * If the user requested a callback for a cyclic DMA in jz4780_dma_begin()
510 for (i = 0; i < jzchan->desc->count; i++) in jz4780_dma_begin()
511 jzchan->desc->desc[i].dcm &= ~JZ_DMA_DCM_LINK; in jz4780_dma_begin()
519 jzchan->curr_hwdesc = in jz4780_dma_begin()
520 (jzchan->curr_hwdesc + 1) % jzchan->desc->count; in jz4780_dma_begin()
524 jz4780_dma_chan_enable(jzdma, jzchan->id); in jz4780_dma_begin()
526 /* Use 4-word descriptors. */ in jz4780_dma_begin()
527 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); in jz4780_dma_begin()
530 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DRT, in jz4780_dma_begin()
531 jzchan->transfer_type); in jz4780_dma_begin()
534 * Set the transfer count. This is redundant for a descriptor-driven in jz4780_dma_begin()
539 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DTC, in jz4780_dma_begin()
540 jzchan->desc->desc[jzchan->curr_hwdesc].dtc); in jz4780_dma_begin()
543 desc_phys = jzchan->desc->desc_phys + in jz4780_dma_begin()
544 (jzchan->curr_hwdesc * sizeof(*jzchan->desc->desc)); in jz4780_dma_begin()
545 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DDA, desc_phys); in jz4780_dma_begin()
546 jz4780_dma_ctrl_writel(jzdma, JZ_DMA_REG_DDRS, BIT(jzchan->id)); in jz4780_dma_begin()
549 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, in jz4780_dma_begin()
558 spin_lock_irqsave(&jzchan->vchan.lock, flags); in jz4780_dma_issue_pending()
560 if (vchan_issue_pending(&jzchan->vchan) && !jzchan->desc) in jz4780_dma_issue_pending()
563 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); in jz4780_dma_issue_pending()
573 spin_lock_irqsave(&jzchan->vchan.lock, flags); in jz4780_dma_terminate_all()
575 /* Clear the DMA status and stop the transfer. */ in jz4780_dma_terminate_all()
576 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); in jz4780_dma_terminate_all()
577 if (jzchan->desc) { in jz4780_dma_terminate_all()
578 vchan_terminate_vdesc(&jzchan->desc->vdesc); in jz4780_dma_terminate_all()
579 jzchan->desc = NULL; in jz4780_dma_terminate_all()
582 jz4780_dma_chan_disable(jzdma, jzchan->id); in jz4780_dma_terminate_all()
584 vchan_get_all_descriptors(&jzchan->vchan, &head); in jz4780_dma_terminate_all()
586 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); in jz4780_dma_terminate_all()
588 vchan_dma_desc_free_list(&jzchan->vchan, &head); in jz4780_dma_terminate_all()
597 vchan_synchronize(&jzchan->vchan); in jz4780_dma_synchronize()
598 jz4780_dma_chan_disable(jzdma, jzchan->id); in jz4780_dma_synchronize()
606 if ((config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES) in jz4780_dma_config()
607 || (config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)) in jz4780_dma_config()
608 return -EINVAL; in jz4780_dma_config()
611 memcpy(&jzchan->config, config, sizeof(jzchan->config)); in jz4780_dma_config()
623 for (i = next_sg; i < desc->count; i++) in jz4780_dma_desc_residue()
624 count += desc->desc[i].dtc & GENMASK(23, 0); in jz4780_dma_desc_residue()
627 count += jz4780_dma_chn_readl(jzdma, jzchan->id, in jz4780_dma_desc_residue()
630 return count << jzchan->transfer_shift; in jz4780_dma_desc_residue()
642 spin_lock_irqsave(&jzchan->vchan.lock, flags); in jz4780_dma_tx_status()
648 vdesc = vchan_find_desc(&jzchan->vchan, cookie); in jz4780_dma_tx_status()
653 } else if (cookie == jzchan->desc->vdesc.tx.cookie) { in jz4780_dma_tx_status()
654 residue = jz4780_dma_desc_residue(jzchan, jzchan->desc, in jz4780_dma_tx_status()
655 jzchan->curr_hwdesc + 1); in jz4780_dma_tx_status()
659 if (vdesc && jzchan->desc && vdesc == &jzchan->desc->vdesc in jz4780_dma_tx_status()
660 && jzchan->desc->status & (JZ_DMA_DCS_AR | JZ_DMA_DCS_HLT)) in jz4780_dma_tx_status()
664 spin_unlock_irqrestore(&jzchan->vchan.lock, flags); in jz4780_dma_tx_status()
671 const unsigned int soc_flags = jzdma->soc_data->flags; in jz4780_dma_chan_irq()
672 struct jz4780_dma_desc *desc = jzchan->desc; in jz4780_dma_chan_irq()
676 spin_lock(&jzchan->vchan.lock); in jz4780_dma_chan_irq()
678 dcs = jz4780_dma_chn_readl(jzdma, jzchan->id, JZ_DMA_REG_DCS); in jz4780_dma_chan_irq()
679 jz4780_dma_chn_writel(jzdma, jzchan->id, JZ_DMA_REG_DCS, 0); in jz4780_dma_chan_irq()
682 dev_warn(&jzchan->vchan.chan.dev->device, in jz4780_dma_chan_irq()
687 dev_warn(&jzchan->vchan.chan.dev->device, in jz4780_dma_chan_irq()
691 if (jzchan->desc) { in jz4780_dma_chan_irq()
692 jzchan->desc->status = dcs; in jz4780_dma_chan_irq()
695 if (jzchan->desc->type == DMA_CYCLIC) { in jz4780_dma_chan_irq()
696 vchan_cyclic_callback(&jzchan->desc->vdesc); in jz4780_dma_chan_irq()
701 (jzchan->curr_hwdesc + 1 == desc->count)) { in jz4780_dma_chan_irq()
702 vchan_cookie_complete(&desc->vdesc); in jz4780_dma_chan_irq()
703 jzchan->desc = NULL; in jz4780_dma_chan_irq()
708 /* False positive - continue the transfer */ in jz4780_dma_chan_irq()
710 jz4780_dma_chn_writel(jzdma, jzchan->id, in jz4780_dma_chan_irq()
716 dev_err(&jzchan->vchan.chan.dev->device, in jz4780_dma_chan_irq()
720 spin_unlock(&jzchan->vchan.lock); in jz4780_dma_chan_irq()
728 unsigned int nb_channels = jzdma->soc_data->nb_channels; in jz4780_dma_irq_handler()
736 if (jz4780_dma_chan_irq(jzdma, &jzdma->chan[i])) in jz4780_dma_irq_handler()
755 jzchan->desc_pool = dma_pool_create(dev_name(&chan->dev->device), in jz4780_dma_alloc_chan_resources()
756 chan->device->dev, in jz4780_dma_alloc_chan_resources()
759 if (!jzchan->desc_pool) { in jz4780_dma_alloc_chan_resources()
760 dev_err(&chan->dev->device, in jz4780_dma_alloc_chan_resources()
762 return -ENOMEM; in jz4780_dma_alloc_chan_resources()
772 vchan_free_chan_resources(&jzchan->vchan); in jz4780_dma_free_chan_resources()
773 dma_pool_destroy(jzchan->desc_pool); in jz4780_dma_free_chan_resources()
774 jzchan->desc_pool = NULL; in jz4780_dma_free_chan_resources()
784 if (data->channel > -1) { in jz4780_dma_filter_fn()
785 if (data->channel != jzchan->id) in jz4780_dma_filter_fn()
787 } else if (jzdma->chan_reserved & BIT(jzchan->id)) { in jz4780_dma_filter_fn()
791 jzchan->transfer_type = data->transfer_type; in jz4780_dma_filter_fn()
799 struct jz4780_dma_dev *jzdma = ofdma->of_dma_data; in jz4780_of_dma_xlate()
800 dma_cap_mask_t mask = jzdma->dma_device.cap_mask; in jz4780_of_dma_xlate()
803 if (dma_spec->args_count != 2) in jz4780_of_dma_xlate()
806 data.transfer_type = dma_spec->args[0]; in jz4780_of_dma_xlate()
807 data.channel = dma_spec->args[1]; in jz4780_of_dma_xlate()
809 if (data.channel > -1) { in jz4780_of_dma_xlate()
810 if (data.channel >= jzdma->soc_data->nb_channels) { in jz4780_of_dma_xlate()
811 dev_err(jzdma->dma_device.dev, in jz4780_of_dma_xlate()
812 "device requested non-existent channel %u\n", in jz4780_of_dma_xlate()
818 if (!(jzdma->chan_reserved & BIT(data.channel))) { in jz4780_of_dma_xlate()
819 dev_err(jzdma->dma_device.dev, in jz4780_of_dma_xlate()
825 jzdma->chan[data.channel].transfer_type = data.transfer_type; in jz4780_of_dma_xlate()
828 &jzdma->chan[data.channel].vchan.chan); in jz4780_of_dma_xlate()
831 ofdma->of_node); in jz4780_of_dma_xlate()
837 struct device *dev = &pdev->dev; in jz4780_dma_probe()
845 if (!dev->of_node) { in jz4780_dma_probe()
847 return -EINVAL; in jz4780_dma_probe()
852 return -EINVAL; in jz4780_dma_probe()
855 soc_data->nb_channels), GFP_KERNEL); in jz4780_dma_probe()
857 return -ENOMEM; in jz4780_dma_probe()
859 jzdma->soc_data = soc_data; in jz4780_dma_probe()
862 jzdma->chn_base = devm_platform_ioremap_resource(pdev, 0); in jz4780_dma_probe()
863 if (IS_ERR(jzdma->chn_base)) in jz4780_dma_probe()
864 return PTR_ERR(jzdma->chn_base); in jz4780_dma_probe()
868 jzdma->ctrl_base = devm_ioremap_resource(dev, res); in jz4780_dma_probe()
869 if (IS_ERR(jzdma->ctrl_base)) in jz4780_dma_probe()
870 return PTR_ERR(jzdma->ctrl_base); in jz4780_dma_probe()
871 } else if (soc_data->flags & JZ_SOC_DATA_ALLOW_LEGACY_DT) { in jz4780_dma_probe()
873 * On JZ4780, if the second memory resource was not supplied, in jz4780_dma_probe()
877 jzdma->ctrl_base = jzdma->chn_base + JZ4780_DMA_CTRL_OFFSET; in jz4780_dma_probe()
880 return -EINVAL; in jz4780_dma_probe()
883 jzdma->clk = devm_clk_get(dev, NULL); in jz4780_dma_probe()
884 if (IS_ERR(jzdma->clk)) { in jz4780_dma_probe()
886 ret = PTR_ERR(jzdma->clk); in jz4780_dma_probe()
890 clk_prepare_enable(jzdma->clk); in jz4780_dma_probe()
893 of_property_read_u32_index(dev->of_node, "ingenic,reserved-channels", in jz4780_dma_probe()
894 0, &jzdma->chan_reserved); in jz4780_dma_probe()
896 dd = &jzdma->dma_device; in jz4780_dma_probe()
898 dma_cap_set(DMA_MEMCPY, dd->cap_mask); in jz4780_dma_probe()
899 dma_cap_set(DMA_SLAVE, dd->cap_mask); in jz4780_dma_probe()
900 dma_cap_set(DMA_CYCLIC, dd->cap_mask); in jz4780_dma_probe()
902 dd->dev = dev; in jz4780_dma_probe()
903 dd->copy_align = DMAENGINE_ALIGN_4_BYTES; in jz4780_dma_probe()
904 dd->device_alloc_chan_resources = jz4780_dma_alloc_chan_resources; in jz4780_dma_probe()
905 dd->device_free_chan_resources = jz4780_dma_free_chan_resources; in jz4780_dma_probe()
906 dd->device_prep_slave_sg = jz4780_dma_prep_slave_sg; in jz4780_dma_probe()
907 dd->device_prep_dma_cyclic = jz4780_dma_prep_dma_cyclic; in jz4780_dma_probe()
908 dd->device_prep_dma_memcpy = jz4780_dma_prep_dma_memcpy; in jz4780_dma_probe()
909 dd->device_config = jz4780_dma_config; in jz4780_dma_probe()
910 dd->device_terminate_all = jz4780_dma_terminate_all; in jz4780_dma_probe()
911 dd->device_synchronize = jz4780_dma_synchronize; in jz4780_dma_probe()
912 dd->device_tx_status = jz4780_dma_tx_status; in jz4780_dma_probe()
913 dd->device_issue_pending = jz4780_dma_issue_pending; in jz4780_dma_probe()
914 dd->src_addr_widths = JZ_DMA_BUSWIDTHS; in jz4780_dma_probe()
915 dd->dst_addr_widths = JZ_DMA_BUSWIDTHS; in jz4780_dma_probe()
916 dd->directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in jz4780_dma_probe()
917 dd->residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in jz4780_dma_probe()
920 * Enable DMA controller, mark all channels as not programmable. in jz4780_dma_probe()
921 * Also set the FMSC bit - it increases MSC performance, so it makes in jz4780_dma_probe()
927 if (soc_data->flags & JZ_SOC_DATA_PROGRAMMABLE_DMA) in jz4780_dma_probe()
930 INIT_LIST_HEAD(&dd->channels); in jz4780_dma_probe()
932 for (i = 0; i < soc_data->nb_channels; i++) { in jz4780_dma_probe()
933 jzchan = &jzdma->chan[i]; in jz4780_dma_probe()
934 jzchan->id = i; in jz4780_dma_probe()
936 vchan_init(&jzchan->vchan, dd); in jz4780_dma_probe()
937 jzchan->vchan.desc_free = jz4780_dma_desc_free; in jz4780_dma_probe()
944 jzdma->irq = ret; in jz4780_dma_probe()
946 ret = request_irq(jzdma->irq, jz4780_dma_irq_handler, 0, dev_name(dev), in jz4780_dma_probe()
949 dev_err(dev, "failed to request IRQ %u!\n", jzdma->irq); in jz4780_dma_probe()
959 /* Register with OF DMA helpers. */ in jz4780_dma_probe()
960 ret = of_dma_controller_register(dev->of_node, jz4780_of_dma_xlate, in jz4780_dma_probe()
963 dev_err(dev, "failed to register OF DMA controller\n"); in jz4780_dma_probe()
967 dev_info(dev, "JZ4780 DMA controller initialised\n"); in jz4780_dma_probe()
971 free_irq(jzdma->irq, jzdma); in jz4780_dma_probe()
974 clk_disable_unprepare(jzdma->clk); in jz4780_dma_probe()
983 of_dma_controller_free(pdev->dev.of_node); in jz4780_dma_remove()
985 clk_disable_unprepare(jzdma->clk); in jz4780_dma_remove()
986 free_irq(jzdma->irq, jzdma); in jz4780_dma_remove()
988 for (i = 0; i < jzdma->soc_data->nb_channels; i++) in jz4780_dma_remove()
989 tasklet_kill(&jzdma->chan[i].vchan.task); in jz4780_dma_remove()
1032 { .compatible = "ingenic,jz4740-dma", .data = &jz4740_dma_soc_data },
1033 { .compatible = "ingenic,jz4725b-dma", .data = &jz4725b_dma_soc_data },
1034 { .compatible = "ingenic,jz4770-dma", .data = &jz4770_dma_soc_data },
1035 { .compatible = "ingenic,jz4780-dma", .data = &jz4780_dma_soc_data },
1036 { .compatible = "ingenic,x1000-dma", .data = &x1000_dma_soc_data },
1037 { .compatible = "ingenic,x1830-dma", .data = &x1830_dma_soc_data },
1046 .name = "jz4780-dma",
1063 MODULE_AUTHOR("Alex Smith <alex@alex-smith.me.uk>");
1064 MODULE_DESCRIPTION("Ingenic JZ4780 DMA controller driver");