77 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)  in to_hidma_dev()  argument
79 	return container_of(dmadev, struct hidma_dev, ddev);  in to_hidma_dev()
93 static void hidma_free(struct hidma_dev *dmadev)  in hidma_free()  argument
95 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_free()
170 	struct hidma_dev *dmadev = to_hidma_dev(ddev);  in hidma_callback()  local
189 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_callback()
190 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_callback()
194 static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig)  in hidma_chan_init()  argument
199 	mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);  in hidma_chan_init()
203 	ddev = &dmadev->ddev;  in hidma_chan_init()
205 	mchan->dmadev = dmadev;  in hidma_chan_init()
217 	dmadev->ddev.chancnt++;  in hidma_chan_init()
223 	struct hidma_dev *dmadev = from_tasklet(dmadev, t, task);  in hidma_issue_task()  local
225 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_issue_task()
226 	hidma_ll_start(dmadev->lldev);  in hidma_issue_task()
232 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_issue_pending()  local
239 		hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch);  in hidma_issue_pending()
252 	status = pm_runtime_get(dmadev->ddev.dev);  in hidma_issue_pending()
254 		tasklet_schedule(&dmadev->task);  in hidma_issue_pending()
256 		hidma_ll_start(dmadev->lldev);  in hidma_issue_pending()
314 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_tx_submit()  local
319 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_tx_submit()
320 	if (!hidma_ll_isenabled(dmadev->lldev)) {  in hidma_tx_submit()
321 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_tx_submit()
322 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_tx_submit()
325 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_tx_submit()
326 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_tx_submit()
345 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_alloc_chan_resources()  local
356 	for (i = 0; i < dmadev->nr_descriptors; i++) {  in hidma_alloc_chan_resources()
365 		rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,  in hidma_alloc_chan_resources()
380 			hidma_ll_free(dmadev->lldev, mdesc->tre_ch);  in hidma_alloc_chan_resources()
399 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memcpy()
432 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memset()
462 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_channel()  local
468 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_terminate_channel()
481 	rc = hidma_ll_disable(dmadev->lldev);  in hidma_terminate_channel()
483 		dev_err(dmadev->ddev.dev, "channel did not pause\n");  in hidma_terminate_channel()
499 	rc = hidma_ll_enable(dmadev->lldev);  in hidma_terminate_channel()
501 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_terminate_channel()
502 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_terminate_channel()
509 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_all()  local
517 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_terminate_all()
518 	rc = hidma_ll_setup(dmadev->lldev);  in hidma_terminate_all()
519 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_terminate_all()
520 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_terminate_all()
527 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_free_chan_resources()
554 	struct hidma_dev *dmadev;  in hidma_pause()  local
557 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_pause()
559 		pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_pause()
560 		if (hidma_ll_disable(dmadev->lldev))  in hidma_pause()
561 			dev_warn(dmadev->ddev.dev, "channel did not stop\n");  in hidma_pause()
563 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_pause()
564 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_pause()
572 	struct hidma_dev *dmadev;  in hidma_resume()  local
576 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_resume()
578 		pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_resume()
579 		rc = hidma_ll_enable(dmadev->lldev);  in hidma_resume()
583 			dev_err(dmadev->ddev.dev,  in hidma_resume()
585 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_resume()
586 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_resume()
606 	struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);  in hidma_chirq_handler_msi()  local
609 				       1 << (chirq - dmadev->msi_virqbase));  in hidma_chirq_handler_msi()
667 	struct hidma_dev *dmadev = dev_get_drvdata(dev);  in hidma_write_msi_msg()  local
670 		writel(msg->address_lo, dmadev->dev_evca + 0x118);  in hidma_write_msi_msg()
671 		writel(msg->address_hi, dmadev->dev_evca + 0x11C);  in hidma_write_msi_msg()
672 		writel(msg->data, dmadev->dev_evca + 0x120);  in hidma_write_msi_msg()
677 static void hidma_free_msis(struct hidma_dev *dmadev)  in hidma_free_msis()  argument
680 	struct device *dev = dmadev->ddev.dev;  in hidma_free_msis()
685 		devm_free_irq(dev, desc->irq, &dmadev->lldev);  in hidma_free_msis()
691 static int hidma_request_msi(struct hidma_dev *dmadev,  in hidma_request_msi()  argument
706 			dmadev->msi_virqbase = desc->irq;  in hidma_request_msi()
711 				       &dmadev->lldev);  in hidma_request_msi()
724 				      &dmadev->lldev);  in hidma_request_msi()
728 		hidma_ll_setup_irq(dmadev->lldev, true);  in hidma_request_msi()
750 	struct hidma_dev *dmadev;  in hidma_probe()  local
788 	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);  in hidma_probe()
789 	if (!dmadev) {  in hidma_probe()
794 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_probe()
795 	spin_lock_init(&dmadev->lock);  in hidma_probe()
796 	dmadev->ddev.dev = &pdev->dev;  in hidma_probe()
797 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_probe()
799 	dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);  in hidma_probe()
800 	dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);  in hidma_probe()
806 	dmadev->dev_evca = evca;  in hidma_probe()
807 	dmadev->evca_resource = evca_resource;  in hidma_probe()
808 	dmadev->dev_trca = trca;  in hidma_probe()
809 	dmadev->trca_resource = trca_resource;  in hidma_probe()
810 	dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;  in hidma_probe()
811 	dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;  in hidma_probe()
812 	dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;  in hidma_probe()
813 	dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;  in hidma_probe()
814 	dmadev->ddev.device_tx_status = hidma_tx_status;  in hidma_probe()
815 	dmadev->ddev.device_issue_pending = hidma_issue_pending;  in hidma_probe()
816 	dmadev->ddev.device_pause = hidma_pause;  in hidma_probe()
817 	dmadev->ddev.device_resume = hidma_resume;  in hidma_probe()
818 	dmadev->ddev.device_terminate_all = hidma_terminate_all;  in hidma_probe()
819 	dmadev->ddev.copy_align = 8;  in hidma_probe()
827 				 &dmadev->nr_descriptors);  in hidma_probe()
832 		dmadev->nr_descriptors = nr_desc_prm;  in hidma_probe()
835 	if (!dmadev->nr_descriptors)  in hidma_probe()
836 		dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;  in hidma_probe()
839 		dmadev->chidx = readl(dmadev->dev_trca + 0x40);  in hidma_probe()
841 		dmadev->chidx = readl(dmadev->dev_trca + 0x28);  in hidma_probe()
852 	dmadev->lldev = hidma_ll_init(dmadev->ddev.dev,  in hidma_probe()
853 				      dmadev->nr_descriptors, dmadev->dev_trca,  in hidma_probe()
854 				      dmadev->dev_evca, dmadev->chidx);  in hidma_probe()
855 	if (!dmadev->lldev) {  in hidma_probe()
860 	platform_set_drvdata(pdev, dmadev);  in hidma_probe()
862 		rc = hidma_request_msi(dmadev, pdev);  in hidma_probe()
865 		hidma_ll_setup_irq(dmadev->lldev, false);  in hidma_probe()
867 				      0, "qcom-hidma", dmadev->lldev);  in hidma_probe()
872 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_probe()
873 	rc = hidma_chan_init(dmadev, 0);  in hidma_probe()
877 	rc = dma_async_device_register(&dmadev->ddev);  in hidma_probe()
881 	dmadev->irq = chirq;  in hidma_probe()
882 	tasklet_setup(&dmadev->task, hidma_issue_task);  in hidma_probe()
883 	hidma_debug_init(dmadev);  in hidma_probe()
884 	hidma_sysfs_init(dmadev);  in hidma_probe()
886 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_probe()
887 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_probe()
892 		hidma_free_msis(dmadev);  in hidma_probe()
894 	hidma_ll_uninit(dmadev->lldev);  in hidma_probe()
896 	if (dmadev)  in hidma_probe()
897 		hidma_free(dmadev);  in hidma_probe()
906 	struct hidma_dev *dmadev = platform_get_drvdata(pdev);  in hidma_shutdown()  local
908 	dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n");  in hidma_shutdown()
910 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_shutdown()
911 	if (hidma_ll_disable(dmadev->lldev))  in hidma_shutdown()
912 		dev_warn(dmadev->ddev.dev, "channel did not stop\n");  in hidma_shutdown()
913 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_shutdown()
914 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_shutdown()
920 	struct hidma_dev *dmadev = platform_get_drvdata(pdev);  in hidma_remove()  local
922 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_remove()
923 	dma_async_device_unregister(&dmadev->ddev);  in hidma_remove()
924 	if (!dmadev->lldev->msi_support)  in hidma_remove()
925 		devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);  in hidma_remove()
927 		hidma_free_msis(dmadev);  in hidma_remove()
929 	tasklet_kill(&dmadev->task);  in hidma_remove()
930 	hidma_sysfs_uninit(dmadev);  in hidma_remove()
931 	hidma_debug_uninit(dmadev);  in hidma_remove()
932 	hidma_ll_uninit(dmadev->lldev);  in hidma_remove()
933 	hidma_free(dmadev);  in hidma_remove()