Lines Matching +full:free +full:- +full:running
4 * Copyright (c) 2015-2017, The Linux Foundation. All rights reserved.
6 * This program is free software; you can redistribute it and/or modify
8 * only version 2 as published by the Free Software Foundation.
24 * written by Hongjun Chen <hong-jun.chen@freescale.com>.
29 * This program is free software; you can redistribute it and/or modify it
30 * under the terms of the GNU General Public License as published by the Free
46 #include <linux/dma-mapping.h>
95 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_free()
110 struct dma_device *ddev = mchan->chan.device; in hidma_process_completed()
122 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
123 list_splice_tail_init(&mchan->completed, &list); in hidma_process_completed()
124 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
132 desc = &mdesc->desc; in hidma_process_completed()
133 last_cookie = desc->cookie; in hidma_process_completed()
135 llstat = hidma_ll_status(mdma->lldev, mdesc->tre_ch); in hidma_process_completed()
137 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
139 mchan->last_success = last_cookie; in hidma_process_completed()
146 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
152 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_process_completed()
153 list_move(&mdesc->node, &mchan->free); in hidma_process_completed()
154 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_process_completed()
168 struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan); in hidma_callback()
169 struct dma_device *ddev = mchan->chan.device; in hidma_callback()
174 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_callback()
175 if (mdesc->node.next) { in hidma_callback()
177 list_move_tail(&mdesc->node, &mchan->completed); in hidma_callback()
180 /* calculate the next running descriptor */ in hidma_callback()
181 mchan->running = list_first_entry(&mchan->active, in hidma_callback()
184 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_callback()
189 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_callback()
190 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_callback()
199 mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL); in hidma_chan_init()
201 return -ENOMEM; in hidma_chan_init()
203 ddev = &dmadev->ddev; in hidma_chan_init()
204 mchan->dma_sig = dma_sig; in hidma_chan_init()
205 mchan->dmadev = dmadev; in hidma_chan_init()
206 mchan->chan.device = ddev; in hidma_chan_init()
207 dma_cookie_init(&mchan->chan); in hidma_chan_init()
209 INIT_LIST_HEAD(&mchan->free); in hidma_chan_init()
210 INIT_LIST_HEAD(&mchan->prepared); in hidma_chan_init()
211 INIT_LIST_HEAD(&mchan->active); in hidma_chan_init()
212 INIT_LIST_HEAD(&mchan->completed); in hidma_chan_init()
213 INIT_LIST_HEAD(&mchan->queued); in hidma_chan_init()
215 spin_lock_init(&mchan->lock); in hidma_chan_init()
216 list_add_tail(&mchan->chan.device_node, &ddev->channels); in hidma_chan_init()
217 dmadev->ddev.chancnt++; in hidma_chan_init()
225 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_issue_task()
226 hidma_ll_start(dmadev->lldev); in hidma_issue_task()
232 struct hidma_dev *dmadev = mchan->dmadev; in hidma_issue_pending()
237 spin_lock_irqsave(&mchan->lock, flags); in hidma_issue_pending()
238 list_for_each_entry_safe(qdesc, next, &mchan->queued, node) { in hidma_issue_pending()
239 hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch); in hidma_issue_pending()
240 list_move_tail(&qdesc->node, &mchan->active); in hidma_issue_pending()
243 if (!mchan->running) { in hidma_issue_pending()
244 struct hidma_desc *desc = list_first_entry(&mchan->active, in hidma_issue_pending()
247 mchan->running = desc; in hidma_issue_pending()
249 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_issue_pending()
252 status = pm_runtime_get(dmadev->ddev.dev); in hidma_issue_pending()
254 tasklet_schedule(&dmadev->task); in hidma_issue_pending()
256 hidma_ll_start(dmadev->lldev); in hidma_issue_pending()
283 is_success = hidma_txn_is_success(cookie, mchan->last_success, in hidma_tx_status()
284 dmach->cookie); in hidma_tx_status()
288 if (mchan->paused && (ret == DMA_IN_PROGRESS)) { in hidma_tx_status()
292 spin_lock_irqsave(&mchan->lock, flags); in hidma_tx_status()
293 if (mchan->running) in hidma_tx_status()
294 runcookie = mchan->running->desc.cookie; in hidma_tx_status()
296 runcookie = -EINVAL; in hidma_tx_status()
301 spin_unlock_irqrestore(&mchan->lock, flags); in hidma_tx_status()
313 struct hidma_chan *mchan = to_hidma_chan(txd->chan); in hidma_tx_submit()
314 struct hidma_dev *dmadev = mchan->dmadev; in hidma_tx_submit()
319 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_tx_submit()
320 if (!hidma_ll_isenabled(dmadev->lldev)) { in hidma_tx_submit()
321 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
322 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
323 return -ENODEV; in hidma_tx_submit()
325 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_tx_submit()
326 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_tx_submit()
329 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_tx_submit()
332 list_move_tail(&mdesc->node, &mchan->queued); in hidma_tx_submit()
337 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_tx_submit()
345 struct hidma_dev *dmadev = mchan->dmadev; in hidma_alloc_chan_resources()
352 if (mchan->allocated) in hidma_alloc_chan_resources()
356 for (i = 0; i < dmadev->nr_descriptors; i++) { in hidma_alloc_chan_resources()
359 rc = -ENOMEM; in hidma_alloc_chan_resources()
362 dma_async_tx_descriptor_init(&mdesc->desc, dmach); in hidma_alloc_chan_resources()
363 mdesc->desc.tx_submit = hidma_tx_submit; in hidma_alloc_chan_resources()
365 rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig, in hidma_alloc_chan_resources()
367 &mdesc->tre_ch); in hidma_alloc_chan_resources()
369 dev_err(dmach->device->dev, in hidma_alloc_chan_resources()
374 list_add_tail(&mdesc->node, &descs); in hidma_alloc_chan_resources()
380 hidma_ll_free(dmadev->lldev, mdesc->tre_ch); in hidma_alloc_chan_resources()
386 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
387 list_splice_tail_init(&descs, &mchan->free); in hidma_alloc_chan_resources()
388 mchan->allocated = true; in hidma_alloc_chan_resources()
389 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_alloc_chan_resources()
399 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memcpy()
402 /* Get free descriptor */ in hidma_prep_dma_memcpy()
403 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
404 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memcpy()
405 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memcpy()
406 list_del(&mdesc->node); in hidma_prep_dma_memcpy()
408 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
413 mdesc->desc.flags = flags; in hidma_prep_dma_memcpy()
414 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memcpy()
419 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
420 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memcpy()
421 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memcpy()
423 return &mdesc->desc; in hidma_prep_dma_memcpy()
432 struct hidma_dev *mdma = mchan->dmadev; in hidma_prep_dma_memset()
436 /* Get free descriptor */ in hidma_prep_dma_memset()
437 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
438 if (!list_empty(&mchan->free)) { in hidma_prep_dma_memset()
439 mdesc = list_first_entry(&mchan->free, struct hidma_desc, node); in hidma_prep_dma_memset()
440 list_del(&mdesc->node); in hidma_prep_dma_memset()
442 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
457 mdesc->desc.flags = flags; in hidma_prep_dma_memset()
458 hidma_ll_set_transfer_params(mdma->lldev, mdesc->tre_ch, in hidma_prep_dma_memset()
463 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_prep_dma_memset()
464 list_add_tail(&mdesc->node, &mchan->prepared); in hidma_prep_dma_memset()
465 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_prep_dma_memset()
467 return &mdesc->desc; in hidma_prep_dma_memset()
473 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_channel()
479 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_channel()
483 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_terminate_channel()
484 mchan->last_success = 0; in hidma_terminate_channel()
485 list_splice_init(&mchan->active, &list); in hidma_terminate_channel()
486 list_splice_init(&mchan->prepared, &list); in hidma_terminate_channel()
487 list_splice_init(&mchan->completed, &list); in hidma_terminate_channel()
488 list_splice_init(&mchan->queued, &list); in hidma_terminate_channel()
489 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_terminate_channel()
492 rc = hidma_ll_disable(dmadev->lldev); in hidma_terminate_channel()
494 dev_err(dmadev->ddev.dev, "channel did not pause\n"); in hidma_terminate_channel()
500 struct dma_async_tx_descriptor *txd = &mdesc->desc; in hidma_terminate_channel()
507 list_move(&mdesc->node, &mchan->free); in hidma_terminate_channel()
510 rc = hidma_ll_enable(dmadev->lldev); in hidma_terminate_channel()
512 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_channel()
513 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_channel()
520 struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device); in hidma_terminate_all()
528 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_terminate_all()
529 rc = hidma_ll_setup(dmadev->lldev); in hidma_terminate_all()
530 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_terminate_all()
531 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_terminate_all()
538 struct hidma_dev *mdma = mchan->dmadev; in hidma_free_chan_resources()
543 /* terminate running transactions and free descriptors */ in hidma_free_chan_resources()
546 spin_lock_irqsave(&mchan->lock, irqflags); in hidma_free_chan_resources()
549 list_splice_tail_init(&mchan->free, &descs); in hidma_free_chan_resources()
551 /* Free descriptors */ in hidma_free_chan_resources()
553 hidma_ll_free(mdma->lldev, mdesc->tre_ch); in hidma_free_chan_resources()
554 list_del(&mdesc->node); in hidma_free_chan_resources()
558 mchan->allocated = false; in hidma_free_chan_resources()
559 spin_unlock_irqrestore(&mchan->lock, irqflags); in hidma_free_chan_resources()
568 dmadev = to_hidma_dev(mchan->chan.device); in hidma_pause()
569 if (!mchan->paused) { in hidma_pause()
570 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_pause()
571 if (hidma_ll_disable(dmadev->lldev)) in hidma_pause()
572 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_pause()
573 mchan->paused = true; in hidma_pause()
574 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_pause()
575 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_pause()
587 dmadev = to_hidma_dev(mchan->chan.device); in hidma_resume()
588 if (mchan->paused) { in hidma_resume()
589 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_resume()
590 rc = hidma_ll_enable(dmadev->lldev); in hidma_resume()
592 mchan->paused = false; in hidma_resume()
594 dev_err(dmadev->ddev.dev, in hidma_resume()
596 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_resume()
597 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_resume()
620 1 << (chirq - dmadev->msi_virqbase)); in hidma_chirq_handler_msi()
631 if (strcmp(attr->attr.name, "chid") == 0) in hidma_show_values()
632 sprintf(buf, "%d\n", mdev->chidx); in hidma_show_values()
639 device_remove_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_uninit()
648 attrs = devm_kmalloc(dev->ddev.dev, sizeof(struct device_attribute), in hidma_create_sysfs_entry()
653 name_copy = devm_kstrdup(dev->ddev.dev, name, GFP_KERNEL); in hidma_create_sysfs_entry()
657 attrs->attr.name = name_copy; in hidma_create_sysfs_entry()
658 attrs->attr.mode = mode; in hidma_create_sysfs_entry()
659 attrs->show = hidma_show_values; in hidma_create_sysfs_entry()
660 sysfs_attr_init(&attrs->attr); in hidma_create_sysfs_entry()
667 dev->chid_attrs = hidma_create_sysfs_entry(dev, "chid", S_IRUGO); in hidma_sysfs_init()
668 if (!dev->chid_attrs) in hidma_sysfs_init()
669 return -ENOMEM; in hidma_sysfs_init()
671 return device_create_file(dev->ddev.dev, dev->chid_attrs); in hidma_sysfs_init()
680 if (!desc->msi_index) { in hidma_write_msi_msg()
681 writel(msg->address_lo, dmadev->dev_evca + 0x118); in hidma_write_msi_msg()
682 writel(msg->address_hi, dmadev->dev_evca + 0x11C); in hidma_write_msi_msg()
683 writel(msg->data, dmadev->dev_evca + 0x120); in hidma_write_msi_msg()
691 struct device *dev = dmadev->ddev.dev; in hidma_free_msis()
697 devm_free_irq(dev, virq, &dmadev->lldev); in hidma_free_msis()
710 rc = platform_msi_domain_alloc_irqs(&pdev->dev, HIDMA_MSI_INTS, in hidma_request_msi()
716 virq = msi_get_virq(&pdev->dev, i); in hidma_request_msi()
717 rc = devm_request_irq(&pdev->dev, virq, in hidma_request_msi()
719 0, "qcom-hidma-msi", in hidma_request_msi()
720 &dmadev->lldev); in hidma_request_msi()
724 dmadev->msi_virqbase = virq; in hidma_request_msi()
728 /* free allocated MSI interrupts above */ in hidma_request_msi()
729 for (--i; i >= 0; i--) { in hidma_request_msi()
730 virq = msi_get_virq(&pdev->dev, i); in hidma_request_msi()
731 devm_free_irq(&pdev->dev, virq, &dmadev->lldev); in hidma_request_msi()
733 dev_warn(&pdev->dev, in hidma_request_msi()
736 /* Add callback to free MSIs on teardown */ in hidma_request_msi()
737 hidma_ll_setup_irq(dmadev->lldev, true); in hidma_request_msi()
741 return -EINVAL; in hidma_request_msi()
764 pm_runtime_set_autosuspend_delay(&pdev->dev, HIDMA_AUTOSUSPEND_TIMEOUT); in hidma_probe()
765 pm_runtime_use_autosuspend(&pdev->dev); in hidma_probe()
766 pm_runtime_set_active(&pdev->dev); in hidma_probe()
767 pm_runtime_enable(&pdev->dev); in hidma_probe()
770 trca = devm_ioremap_resource(&pdev->dev, trca_resource); in hidma_probe()
772 rc = -ENOMEM; in hidma_probe()
777 evca = devm_ioremap_resource(&pdev->dev, evca_resource); in hidma_probe()
779 rc = -ENOMEM; in hidma_probe()
789 rc = -ENODEV; in hidma_probe()
793 dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL); in hidma_probe()
795 rc = -ENOMEM; in hidma_probe()
799 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
800 spin_lock_init(&dmadev->lock); in hidma_probe()
801 dmadev->ddev.dev = &pdev->dev; in hidma_probe()
802 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_probe()
804 dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask); in hidma_probe()
805 dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask); in hidma_probe()
806 if (WARN_ON(!pdev->dev.dma_mask)) { in hidma_probe()
807 rc = -ENXIO; in hidma_probe()
811 dmadev->dev_evca = evca; in hidma_probe()
812 dmadev->evca_resource = evca_resource; in hidma_probe()
813 dmadev->dev_trca = trca; in hidma_probe()
814 dmadev->trca_resource = trca_resource; in hidma_probe()
815 dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy; in hidma_probe()
816 dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset; in hidma_probe()
817 dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources; in hidma_probe()
818 dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources; in hidma_probe()
819 dmadev->ddev.device_tx_status = hidma_tx_status; in hidma_probe()
820 dmadev->ddev.device_issue_pending = hidma_issue_pending; in hidma_probe()
821 dmadev->ddev.device_pause = hidma_pause; in hidma_probe()
822 dmadev->ddev.device_resume = hidma_resume; in hidma_probe()
823 dmadev->ddev.device_terminate_all = hidma_terminate_all; in hidma_probe()
824 dmadev->ddev.copy_align = 8; in hidma_probe()
830 msi = hidma_test_capability(&pdev->dev, HIDMA_MSI_CAP); in hidma_probe()
831 device_property_read_u32(&pdev->dev, "desc-count", in hidma_probe()
832 &dmadev->nr_descriptors); in hidma_probe()
835 dev_info(&pdev->dev, "overriding number of descriptors as %d\n", in hidma_probe()
837 dmadev->nr_descriptors = nr_desc_prm; in hidma_probe()
840 if (!dmadev->nr_descriptors) in hidma_probe()
841 dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC; in hidma_probe()
843 if (hidma_test_capability(&pdev->dev, HIDMA_IDENTITY_CAP)) in hidma_probe()
844 dmadev->chidx = readl(dmadev->dev_trca + 0x40); in hidma_probe()
846 dmadev->chidx = readl(dmadev->dev_trca + 0x28); in hidma_probe()
849 rc = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); in hidma_probe()
851 dev_warn(&pdev->dev, "unable to set coherent mask to 64"); in hidma_probe()
855 dmadev->lldev = hidma_ll_init(dmadev->ddev.dev, in hidma_probe()
856 dmadev->nr_descriptors, dmadev->dev_trca, in hidma_probe()
857 dmadev->dev_evca, dmadev->chidx); in hidma_probe()
858 if (!dmadev->lldev) { in hidma_probe()
859 rc = -EPROBE_DEFER; in hidma_probe()
868 hidma_ll_setup_irq(dmadev->lldev, false); in hidma_probe()
869 rc = devm_request_irq(&pdev->dev, chirq, hidma_chirq_handler, in hidma_probe()
870 0, "qcom-hidma", dmadev->lldev); in hidma_probe()
875 INIT_LIST_HEAD(&dmadev->ddev.channels); in hidma_probe()
880 rc = dma_async_device_register(&dmadev->ddev); in hidma_probe()
884 dmadev->irq = chirq; in hidma_probe()
885 tasklet_setup(&dmadev->task, hidma_issue_task); in hidma_probe()
888 dev_info(&pdev->dev, "HI-DMA engine driver registration complete\n"); in hidma_probe()
889 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_probe()
890 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_probe()
897 hidma_ll_uninit(dmadev->lldev); in hidma_probe()
902 pm_runtime_put_sync(&pdev->dev); in hidma_probe()
903 pm_runtime_disable(&pdev->dev); in hidma_probe()
911 dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n"); in hidma_shutdown()
913 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_shutdown()
914 if (hidma_ll_disable(dmadev->lldev)) in hidma_shutdown()
915 dev_warn(dmadev->ddev.dev, "channel did not stop\n"); in hidma_shutdown()
916 pm_runtime_mark_last_busy(dmadev->ddev.dev); in hidma_shutdown()
917 pm_runtime_put_autosuspend(dmadev->ddev.dev); in hidma_shutdown()
925 pm_runtime_get_sync(dmadev->ddev.dev); in hidma_remove()
926 dma_async_device_unregister(&dmadev->ddev); in hidma_remove()
927 if (!dmadev->lldev->msi_support) in hidma_remove()
928 devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev); in hidma_remove()
932 tasklet_kill(&dmadev->task); in hidma_remove()
935 hidma_ll_uninit(dmadev->lldev); in hidma_remove()
938 dev_info(&pdev->dev, "HI-DMA engine removed\n"); in hidma_remove()
939 pm_runtime_put_sync_suspend(&pdev->dev); in hidma_remove()
940 pm_runtime_disable(&pdev->dev); in hidma_remove()
956 {.compatible = "qcom,hidma-1.0",},
957 {.compatible = "qcom,hidma-1.1", .data = (void *)(HIDMA_MSI_CAP),},
958 {.compatible = "qcom,hidma-1.2",