77 static inline struct hidma_dev *to_hidma_dev(struct dma_device *dmadev)  in to_hidma_dev()  argument
79 	return container_of(dmadev, struct hidma_dev, ddev);  in to_hidma_dev()
93 static void hidma_free(struct hidma_dev *dmadev)  in hidma_free()  argument
95 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_free()
170 	struct hidma_dev *dmadev = to_hidma_dev(ddev);  in hidma_callback()  local
189 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_callback()
190 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_callback()
194 static int hidma_chan_init(struct hidma_dev *dmadev, u32 dma_sig)  in hidma_chan_init()  argument
199 	mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);  in hidma_chan_init()
203 	ddev = &dmadev->ddev;  in hidma_chan_init()
205 	mchan->dmadev = dmadev;  in hidma_chan_init()
217 	dmadev->ddev.chancnt++;  in hidma_chan_init()
223 	struct hidma_dev *dmadev = from_tasklet(dmadev, t, task);  in hidma_issue_task()  local
225 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_issue_task()
226 	hidma_ll_start(dmadev->lldev);  in hidma_issue_task()
232 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_issue_pending()  local
239 		hidma_ll_queue_request(dmadev->lldev, qdesc->tre_ch);  in hidma_issue_pending()
252 	status = pm_runtime_get(dmadev->ddev.dev);  in hidma_issue_pending()
254 		tasklet_schedule(&dmadev->task);  in hidma_issue_pending()
256 		hidma_ll_start(dmadev->lldev);  in hidma_issue_pending()
314 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_tx_submit()  local
319 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_tx_submit()
320 	if (!hidma_ll_isenabled(dmadev->lldev)) {  in hidma_tx_submit()
321 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_tx_submit()
322 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_tx_submit()
325 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_tx_submit()
326 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_tx_submit()
345 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_alloc_chan_resources()  local
356 	for (i = 0; i < dmadev->nr_descriptors; i++) {  in hidma_alloc_chan_resources()
365 		rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,  in hidma_alloc_chan_resources()
380 			hidma_ll_free(dmadev->lldev, mdesc->tre_ch);  in hidma_alloc_chan_resources()
399 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memcpy()
432 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memset()
473 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_channel()  local
479 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_terminate_channel()
492 	rc = hidma_ll_disable(dmadev->lldev);  in hidma_terminate_channel()
494 		dev_err(dmadev->ddev.dev, "channel did not pause\n");  in hidma_terminate_channel()
510 	rc = hidma_ll_enable(dmadev->lldev);  in hidma_terminate_channel()
512 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_terminate_channel()
513 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_terminate_channel()
520 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_all()  local
528 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_terminate_all()
529 	rc = hidma_ll_setup(dmadev->lldev);  in hidma_terminate_all()
530 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_terminate_all()
531 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_terminate_all()
538 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_free_chan_resources()
565 	struct hidma_dev *dmadev;  in hidma_pause()  local
568 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_pause()
570 		pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_pause()
571 		if (hidma_ll_disable(dmadev->lldev))  in hidma_pause()
572 			dev_warn(dmadev->ddev.dev, "channel did not stop\n");  in hidma_pause()
574 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_pause()
575 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_pause()
583 	struct hidma_dev *dmadev;  in hidma_resume()  local
587 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_resume()
589 		pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_resume()
590 		rc = hidma_ll_enable(dmadev->lldev);  in hidma_resume()
594 			dev_err(dmadev->ddev.dev,  in hidma_resume()
596 		pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_resume()
597 		pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_resume()
617 	struct hidma_dev *dmadev = to_hidma_dev_from_lldev(lldevp);  in hidma_chirq_handler_msi()  local
620 				       1 << (chirq - dmadev->msi_virqbase));  in hidma_chirq_handler_msi()
678 	struct hidma_dev *dmadev = dev_get_drvdata(dev);  in hidma_write_msi_msg()  local
681 		writel(msg->address_lo, dmadev->dev_evca + 0x118);  in hidma_write_msi_msg()
682 		writel(msg->address_hi, dmadev->dev_evca + 0x11C);  in hidma_write_msi_msg()
683 		writel(msg->data, dmadev->dev_evca + 0x120);  in hidma_write_msi_msg()
688 static void hidma_free_msis(struct hidma_dev *dmadev)  in hidma_free_msis()  argument
691 	struct device *dev = dmadev->ddev.dev;  in hidma_free_msis()
697 			devm_free_irq(dev, virq, &dmadev->lldev);  in hidma_free_msis()
704 static int hidma_request_msi(struct hidma_dev *dmadev,  in hidma_request_msi()  argument
720 				       &dmadev->lldev);  in hidma_request_msi()
724 			dmadev->msi_virqbase = virq;  in hidma_request_msi()
731 			devm_free_irq(&pdev->dev, virq, &dmadev->lldev);  in hidma_request_msi()
737 		hidma_ll_setup_irq(dmadev->lldev, true);  in hidma_request_msi()
755 	struct hidma_dev *dmadev;  in hidma_probe()  local
793 	dmadev = devm_kzalloc(&pdev->dev, sizeof(*dmadev), GFP_KERNEL);  in hidma_probe()
794 	if (!dmadev) {  in hidma_probe()
799 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_probe()
800 	spin_lock_init(&dmadev->lock);  in hidma_probe()
801 	dmadev->ddev.dev = &pdev->dev;  in hidma_probe()
802 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_probe()
804 	dma_cap_set(DMA_MEMCPY, dmadev->ddev.cap_mask);  in hidma_probe()
805 	dma_cap_set(DMA_MEMSET, dmadev->ddev.cap_mask);  in hidma_probe()
811 	dmadev->dev_evca = evca;  in hidma_probe()
812 	dmadev->evca_resource = evca_resource;  in hidma_probe()
813 	dmadev->dev_trca = trca;  in hidma_probe()
814 	dmadev->trca_resource = trca_resource;  in hidma_probe()
815 	dmadev->ddev.device_prep_dma_memcpy = hidma_prep_dma_memcpy;  in hidma_probe()
816 	dmadev->ddev.device_prep_dma_memset = hidma_prep_dma_memset;  in hidma_probe()
817 	dmadev->ddev.device_alloc_chan_resources = hidma_alloc_chan_resources;  in hidma_probe()
818 	dmadev->ddev.device_free_chan_resources = hidma_free_chan_resources;  in hidma_probe()
819 	dmadev->ddev.device_tx_status = hidma_tx_status;  in hidma_probe()
820 	dmadev->ddev.device_issue_pending = hidma_issue_pending;  in hidma_probe()
821 	dmadev->ddev.device_pause = hidma_pause;  in hidma_probe()
822 	dmadev->ddev.device_resume = hidma_resume;  in hidma_probe()
823 	dmadev->ddev.device_terminate_all = hidma_terminate_all;  in hidma_probe()
824 	dmadev->ddev.copy_align = 8;  in hidma_probe()
832 				 &dmadev->nr_descriptors);  in hidma_probe()
837 		dmadev->nr_descriptors = nr_desc_prm;  in hidma_probe()
840 	if (!dmadev->nr_descriptors)  in hidma_probe()
841 		dmadev->nr_descriptors = HIDMA_NR_DEFAULT_DESC;  in hidma_probe()
844 		dmadev->chidx = readl(dmadev->dev_trca + 0x40);  in hidma_probe()
846 		dmadev->chidx = readl(dmadev->dev_trca + 0x28);  in hidma_probe()
855 	dmadev->lldev = hidma_ll_init(dmadev->ddev.dev,  in hidma_probe()
856 				      dmadev->nr_descriptors, dmadev->dev_trca,  in hidma_probe()
857 				      dmadev->dev_evca, dmadev->chidx);  in hidma_probe()
858 	if (!dmadev->lldev) {  in hidma_probe()
863 	platform_set_drvdata(pdev, dmadev);  in hidma_probe()
865 		rc = hidma_request_msi(dmadev, pdev);  in hidma_probe()
868 		hidma_ll_setup_irq(dmadev->lldev, false);  in hidma_probe()
870 				      0, "qcom-hidma", dmadev->lldev);  in hidma_probe()
875 	INIT_LIST_HEAD(&dmadev->ddev.channels);  in hidma_probe()
876 	rc = hidma_chan_init(dmadev, 0);  in hidma_probe()
880 	rc = dma_async_device_register(&dmadev->ddev);  in hidma_probe()
884 	dmadev->irq = chirq;  in hidma_probe()
885 	tasklet_setup(&dmadev->task, hidma_issue_task);  in hidma_probe()
886 	hidma_debug_init(dmadev);  in hidma_probe()
887 	hidma_sysfs_init(dmadev);  in hidma_probe()
889 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_probe()
890 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_probe()
895 		hidma_free_msis(dmadev);  in hidma_probe()
897 	hidma_ll_uninit(dmadev->lldev);  in hidma_probe()
899 	if (dmadev)  in hidma_probe()
900 		hidma_free(dmadev);  in hidma_probe()
909 	struct hidma_dev *dmadev = platform_get_drvdata(pdev);  in hidma_shutdown()  local
911 	dev_info(dmadev->ddev.dev, "HI-DMA engine shutdown\n");  in hidma_shutdown()
913 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_shutdown()
914 	if (hidma_ll_disable(dmadev->lldev))  in hidma_shutdown()
915 		dev_warn(dmadev->ddev.dev, "channel did not stop\n");  in hidma_shutdown()
916 	pm_runtime_mark_last_busy(dmadev->ddev.dev);  in hidma_shutdown()
917 	pm_runtime_put_autosuspend(dmadev->ddev.dev);  in hidma_shutdown()
923 	struct hidma_dev *dmadev = platform_get_drvdata(pdev);  in hidma_remove()  local
925 	pm_runtime_get_sync(dmadev->ddev.dev);  in hidma_remove()
926 	dma_async_device_unregister(&dmadev->ddev);  in hidma_remove()
927 	if (!dmadev->lldev->msi_support)  in hidma_remove()
928 		devm_free_irq(dmadev->ddev.dev, dmadev->irq, dmadev->lldev);  in hidma_remove()
930 		hidma_free_msis(dmadev);  in hidma_remove()
932 	tasklet_kill(&dmadev->task);  in hidma_remove()
933 	hidma_sysfs_uninit(dmadev);  in hidma_remove()
934 	hidma_debug_uninit(dmadev);  in hidma_remove()
935 	hidma_ll_uninit(dmadev->lldev);  in hidma_remove()
936 	hidma_free(dmadev);  in hidma_remove()