264 static struct device *hsdma2dev(struct mtk_hsdma_device *hsdma)  in hsdma2dev()  argument
266 	return hsdma->ddev.dev;  in hsdma2dev()
269 static u32 mtk_dma_read(struct mtk_hsdma_device *hsdma, u32 reg)  in mtk_dma_read()  argument
271 	return readl(hsdma->base + reg);  in mtk_dma_read()
274 static void mtk_dma_write(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_write()  argument
276 	writel(val, hsdma->base + reg);  in mtk_dma_write()
279 static void mtk_dma_rmw(struct mtk_hsdma_device *hsdma, u32 reg,  in mtk_dma_rmw()  argument
284 	val = mtk_dma_read(hsdma, reg);  in mtk_dma_rmw()
287 	mtk_dma_write(hsdma, reg, val);  in mtk_dma_rmw()
290 static void mtk_dma_set(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_set()  argument
292 	mtk_dma_rmw(hsdma, reg, 0, val);  in mtk_dma_set()
295 static void mtk_dma_clr(struct mtk_hsdma_device *hsdma, u32 reg, u32 val)  in mtk_dma_clr()  argument
297 	mtk_dma_rmw(hsdma, reg, val, 0);  in mtk_dma_clr()
305 static int mtk_hsdma_busy_wait(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_busy_wait()  argument
309 	return readl_poll_timeout(hsdma->base + MTK_HSDMA_GLO, status,  in mtk_hsdma_busy_wait()
315 static int mtk_hsdma_alloc_pchan(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_alloc_pchan()  argument
328 	ring->txd = dma_zalloc_coherent(hsdma2dev(hsdma), pc->sz_ring,  in mtk_hsdma_alloc_pchan()
347 	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_alloc_pchan()
348 	err = mtk_hsdma_busy_wait(hsdma);  in mtk_hsdma_alloc_pchan()
353 	mtk_dma_set(hsdma, MTK_HSDMA_RESET,  in mtk_hsdma_alloc_pchan()
355 	mtk_dma_clr(hsdma, MTK_HSDMA_RESET,  in mtk_hsdma_alloc_pchan()
359 	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, ring->tphys);  in mtk_hsdma_alloc_pchan()
360 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, MTK_DMA_SIZE);  in mtk_hsdma_alloc_pchan()
361 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);  in mtk_hsdma_alloc_pchan()
362 	mtk_dma_write(hsdma, MTK_HSDMA_TX_DMA, 0);  in mtk_hsdma_alloc_pchan()
363 	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, ring->rphys);  in mtk_hsdma_alloc_pchan()
364 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, MTK_DMA_SIZE);  in mtk_hsdma_alloc_pchan()
365 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, ring->cur_rptr);  in mtk_hsdma_alloc_pchan()
366 	mtk_dma_write(hsdma, MTK_HSDMA_RX_DMA, 0);  in mtk_hsdma_alloc_pchan()
369 	mtk_dma_set(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_alloc_pchan()
372 	mtk_dma_write(hsdma, MTK_HSDMA_DLYINT, MTK_HSDMA_DLYINT_DEFAULT);  in mtk_hsdma_alloc_pchan()
375 	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_alloc_pchan()
383 	dma_free_coherent(hsdma2dev(hsdma),  in mtk_hsdma_alloc_pchan()
388 static void mtk_hsdma_free_pchan(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_free_pchan()  argument
394 	mtk_dma_clr(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DMA);  in mtk_hsdma_free_pchan()
395 	mtk_hsdma_busy_wait(hsdma);  in mtk_hsdma_free_pchan()
398 	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_free_pchan()
399 	mtk_dma_write(hsdma, MTK_HSDMA_TX_BASE, 0);  in mtk_hsdma_free_pchan()
400 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CNT, 0);  in mtk_hsdma_free_pchan()
401 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, 0);  in mtk_hsdma_free_pchan()
402 	mtk_dma_write(hsdma, MTK_HSDMA_RX_BASE, 0);  in mtk_hsdma_free_pchan()
403 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CNT, 0);  in mtk_hsdma_free_pchan()
404 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, MTK_DMA_SIZE - 1);  in mtk_hsdma_free_pchan()
408 	dma_free_coherent(hsdma2dev(hsdma),  in mtk_hsdma_free_pchan()
412 static int mtk_hsdma_issue_pending_vdesc(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_issue_pending_vdesc()  argument
422 	spin_lock_irqsave(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
432 		spin_unlock_irqrestore(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
453 			   hsdma->soc->ls0 | MTK_HSDMA_DESC_PLEN(tlen));  in mtk_hsdma_issue_pending_vdesc()
488 	mtk_dma_write(hsdma, MTK_HSDMA_TX_CPU, ring->cur_tptr);  in mtk_hsdma_issue_pending_vdesc()
490 	spin_unlock_irqrestore(&hsdma->lock, flags);  in mtk_hsdma_issue_pending_vdesc()
495 static void mtk_hsdma_issue_vchan_pending(struct mtk_hsdma_device *hsdma,  in mtk_hsdma_issue_vchan_pending()  argument
509 		err = mtk_hsdma_issue_pending_vdesc(hsdma, hsdma->pc, hvd);  in mtk_hsdma_issue_vchan_pending()
533 static void mtk_hsdma_free_rooms_in_ring(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_free_rooms_in_ring()  argument
546 	status = mtk_dma_read(hsdma, MTK_HSDMA_INT_STATUS);  in mtk_hsdma_free_rooms_in_ring()
550 	pc = hsdma->pc;  in mtk_hsdma_free_rooms_in_ring()
569 		if (!(desc2 & hsdma->soc->ddone))  in mtk_hsdma_free_rooms_in_ring()
574 			dev_err(hsdma2dev(hsdma), "cb->vd cannot be null\n");  in mtk_hsdma_free_rooms_in_ring()
622 	mtk_dma_write(hsdma, MTK_HSDMA_RX_CPU, pc->ring.cur_rptr);  in mtk_hsdma_free_rooms_in_ring()
630 		mtk_dma_write(hsdma, MTK_HSDMA_INT_STATUS, status);  in mtk_hsdma_free_rooms_in_ring()
633 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_free_rooms_in_ring()
634 		hvc = &hsdma->vc[i];  in mtk_hsdma_free_rooms_in_ring()
636 		mtk_hsdma_issue_vchan_pending(hsdma, hvc);  in mtk_hsdma_free_rooms_in_ring()
642 	mtk_dma_set(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_free_rooms_in_ring()
647 	struct mtk_hsdma_device *hsdma = devid;  in mtk_hsdma_irq()  local
653 	mtk_dma_clr(hsdma, MTK_HSDMA_INT_ENABLE, MTK_HSDMA_INT_RXDONE);  in mtk_hsdma_irq()
655 	mtk_hsdma_free_rooms_in_ring(hsdma);  in mtk_hsdma_irq()
708 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_issue_pending()  local
715 		mtk_hsdma_issue_vchan_pending(hsdma, hvc);  in mtk_hsdma_issue_pending()
810 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_alloc_chan_resources()  local
818 	if (!refcount_read(&hsdma->pc_refcnt)) {  in mtk_hsdma_alloc_chan_resources()
819 		err = mtk_hsdma_alloc_pchan(hsdma, hsdma->pc);  in mtk_hsdma_alloc_chan_resources()
826 		refcount_set(&hsdma->pc_refcnt, 1);  in mtk_hsdma_alloc_chan_resources()
828 		refcount_inc(&hsdma->pc_refcnt);  in mtk_hsdma_alloc_chan_resources()
836 	struct mtk_hsdma_device *hsdma = to_hsdma_dev(c);  in mtk_hsdma_free_chan_resources()  local
842 	if (!refcount_dec_and_test(&hsdma->pc_refcnt))  in mtk_hsdma_free_chan_resources()
845 	mtk_hsdma_free_pchan(hsdma, hsdma->pc);  in mtk_hsdma_free_chan_resources()
848 static int mtk_hsdma_hw_init(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_hw_init()  argument
852 	pm_runtime_enable(hsdma2dev(hsdma));  in mtk_hsdma_hw_init()
853 	pm_runtime_get_sync(hsdma2dev(hsdma));  in mtk_hsdma_hw_init()
855 	err = clk_prepare_enable(hsdma->clk);  in mtk_hsdma_hw_init()
859 	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);  in mtk_hsdma_hw_init()
860 	mtk_dma_write(hsdma, MTK_HSDMA_GLO, MTK_HSDMA_GLO_DEFAULT);  in mtk_hsdma_hw_init()
865 static int mtk_hsdma_hw_deinit(struct mtk_hsdma_device *hsdma)  in mtk_hsdma_hw_deinit()  argument
867 	mtk_dma_write(hsdma, MTK_HSDMA_GLO, 0);  in mtk_hsdma_hw_deinit()
869 	clk_disable_unprepare(hsdma->clk);  in mtk_hsdma_hw_deinit()
871 	pm_runtime_put_sync(hsdma2dev(hsdma));  in mtk_hsdma_hw_deinit()
872 	pm_runtime_disable(hsdma2dev(hsdma));  in mtk_hsdma_hw_deinit()
896 	struct mtk_hsdma_device *hsdma;  in mtk_hsdma_probe()  local
902 	hsdma = devm_kzalloc(&pdev->dev, sizeof(*hsdma), GFP_KERNEL);  in mtk_hsdma_probe()
903 	if (!hsdma)  in mtk_hsdma_probe()
906 	dd = &hsdma->ddev;  in mtk_hsdma_probe()
909 	hsdma->base = devm_ioremap_resource(&pdev->dev, res);  in mtk_hsdma_probe()
910 	if (IS_ERR(hsdma->base))  in mtk_hsdma_probe()
911 		return PTR_ERR(hsdma->base);  in mtk_hsdma_probe()
913 	hsdma->soc = of_device_get_match_data(&pdev->dev);  in mtk_hsdma_probe()
914 	if (!hsdma->soc) {  in mtk_hsdma_probe()
919 	hsdma->clk = devm_clk_get(&pdev->dev, "hsdma");  in mtk_hsdma_probe()
920 	if (IS_ERR(hsdma->clk)) {  in mtk_hsdma_probe()
923 		return PTR_ERR(hsdma->clk);  in mtk_hsdma_probe()
932 	hsdma->irq = res->start;  in mtk_hsdma_probe()
934 	refcount_set(&hsdma->pc_refcnt, 0);  in mtk_hsdma_probe()
935 	spin_lock_init(&hsdma->lock);  in mtk_hsdma_probe()
953 	hsdma->dma_requests = MTK_HSDMA_NR_VCHANS;  in mtk_hsdma_probe()
956 						      &hsdma->dma_requests)) {  in mtk_hsdma_probe()
962 	hsdma->pc = devm_kcalloc(&pdev->dev, MTK_HSDMA_NR_MAX_PCHANS,  in mtk_hsdma_probe()
963 				 sizeof(*hsdma->pc), GFP_KERNEL);  in mtk_hsdma_probe()
964 	if (!hsdma->pc)  in mtk_hsdma_probe()
967 	hsdma->vc = devm_kcalloc(&pdev->dev, hsdma->dma_requests,  in mtk_hsdma_probe()
968 				 sizeof(*hsdma->vc), GFP_KERNEL);  in mtk_hsdma_probe()
969 	if (!hsdma->vc)  in mtk_hsdma_probe()
972 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_probe()
973 		vc = &hsdma->vc[i];  in mtk_hsdma_probe()
985 					 of_dma_xlate_by_chan_id, hsdma);  in mtk_hsdma_probe()
992 	mtk_hsdma_hw_init(hsdma);  in mtk_hsdma_probe()
994 	err = devm_request_irq(&pdev->dev, hsdma->irq,  in mtk_hsdma_probe()
996 			       dev_name(&pdev->dev), hsdma);  in mtk_hsdma_probe()
1003 	platform_set_drvdata(pdev, hsdma);  in mtk_hsdma_probe()
1017 	struct mtk_hsdma_device *hsdma = platform_get_drvdata(pdev);  in mtk_hsdma_remove()  local
1022 	for (i = 0; i < hsdma->dma_requests; i++) {  in mtk_hsdma_remove()
1023 		vc = &hsdma->vc[i];  in mtk_hsdma_remove()
1030 	mtk_dma_write(hsdma, MTK_HSDMA_INT_ENABLE, 0);  in mtk_hsdma_remove()
1033 	synchronize_irq(hsdma->irq);  in mtk_hsdma_remove()
1036 	mtk_hsdma_hw_deinit(hsdma);  in mtk_hsdma_remove()
1038 	dma_async_device_unregister(&hsdma->ddev);  in mtk_hsdma_remove()