187 static void mv_xor_v2_set_data_buffers(struct mv_xor_v2_device *xor_dev,  in mv_xor_v2_set_data_buffers()  argument
223 static void mv_xor_v2_add_desc_to_desq(struct mv_xor_v2_device *xor_dev,  in mv_xor_v2_add_desc_to_desq()  argument
227 	writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ADD_OFF);  in mv_xor_v2_add_desc_to_desq()
233 static void mv_xor_v2_free_desc_from_desq(struct mv_xor_v2_device *xor_dev,  in mv_xor_v2_free_desc_from_desq()  argument
237 	writel(num_of_desc, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DEALLOC_OFF);  in mv_xor_v2_free_desc_from_desq()
244 static int mv_xor_v2_set_desc_size(struct mv_xor_v2_device *xor_dev)  in mv_xor_v2_set_desc_size()  argument
247 	       xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_CTRL_OFF);  in mv_xor_v2_set_desc_size()
256 void mv_xor_v2_enable_imsg_thrd(struct mv_xor_v2_device *xor_dev)  in mv_xor_v2_enable_imsg_thrd()  argument
261 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);  in mv_xor_v2_enable_imsg_thrd()
265 	writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_THRD_OFF);  in mv_xor_v2_enable_imsg_thrd()
268 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);  in mv_xor_v2_enable_imsg_thrd()
271 	writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_TMOT);  in mv_xor_v2_enable_imsg_thrd()
276 	struct mv_xor_v2_device *xor_dev = data;  in mv_xor_v2_interrupt_handler()  local
280 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);  in mv_xor_v2_interrupt_handler()
290 	tasklet_schedule(&xor_dev->irq_tasklet);  in mv_xor_v2_interrupt_handler()
305 	struct mv_xor_v2_device *xor_dev =  in mv_xor_v2_tx_submit()  local
308 	dev_dbg(xor_dev->dmadev.dev,  in mv_xor_v2_tx_submit()
313 	spin_lock_bh(&xor_dev->lock);  in mv_xor_v2_tx_submit()
317 	dest_hw_desc = xor_dev->hw_desq_virt + xor_dev->hw_queue_idx;  in mv_xor_v2_tx_submit()
319 	memcpy(dest_hw_desc, &sw_desc->hw_desc, xor_dev->desc_size);  in mv_xor_v2_tx_submit()
321 	xor_dev->npendings++;  in mv_xor_v2_tx_submit()
322 	xor_dev->hw_queue_idx++;  in mv_xor_v2_tx_submit()
323 	if (xor_dev->hw_queue_idx >= MV_XOR_V2_DESC_NUM)  in mv_xor_v2_tx_submit()
324 		xor_dev->hw_queue_idx = 0;  in mv_xor_v2_tx_submit()
326 	spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_tx_submit()
335 mv_xor_v2_prep_sw_desc(struct mv_xor_v2_device *xor_dev)  in mv_xor_v2_prep_sw_desc()  argument
341 	spin_lock_bh(&xor_dev->lock);  in mv_xor_v2_prep_sw_desc()
343 	if (list_empty(&xor_dev->free_sw_desc)) {  in mv_xor_v2_prep_sw_desc()
344 		spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_prep_sw_desc()
346 		tasklet_schedule(&xor_dev->irq_tasklet);  in mv_xor_v2_prep_sw_desc()
350 	list_for_each_entry(sw_desc, &xor_dev->free_sw_desc, free_list) {  in mv_xor_v2_prep_sw_desc()
358 		spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_prep_sw_desc()
365 	spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_prep_sw_desc()
379 	struct mv_xor_v2_device	*xor_dev;  in mv_xor_v2_prep_dma_memcpy()  local
381 	xor_dev = container_of(chan, struct mv_xor_v2_device, dmachan);  in mv_xor_v2_prep_dma_memcpy()
383 	dev_dbg(xor_dev->dmadev.dev,  in mv_xor_v2_prep_dma_memcpy()
387 	sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);  in mv_xor_v2_prep_dma_memcpy()
432 	struct mv_xor_v2_device	*xor_dev =  in mv_xor_v2_prep_dma_xor()  local
439 	dev_dbg(xor_dev->dmadev.dev,  in mv_xor_v2_prep_dma_xor()
443 	sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);  in mv_xor_v2_prep_dma_xor()
465 		mv_xor_v2_set_data_buffers(xor_dev, hw_descriptor, src[i], i);  in mv_xor_v2_prep_dma_xor()
490 	struct mv_xor_v2_device	*xor_dev =  in mv_xor_v2_prep_dma_interrupt()  local
493 	sw_desc = mv_xor_v2_prep_sw_desc(xor_dev);  in mv_xor_v2_prep_dma_interrupt()
517 	struct mv_xor_v2_device *xor_dev =  in mv_xor_v2_issue_pending()  local
520 	spin_lock_bh(&xor_dev->lock);  in mv_xor_v2_issue_pending()
526 	mv_xor_v2_add_desc_to_desq(xor_dev, xor_dev->npendings);  in mv_xor_v2_issue_pending()
527 	xor_dev->npendings = 0;  in mv_xor_v2_issue_pending()
529 	spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_issue_pending()
533 int mv_xor_v2_get_pending_params(struct mv_xor_v2_device *xor_dev,  in mv_xor_v2_get_pending_params()  argument
538 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_DONE_OFF);  in mv_xor_v2_get_pending_params()
554 	struct mv_xor_v2_device *xor_dev = (struct mv_xor_v2_device *) data;  in mv_xor_v2_tasklet()  local
558 	dev_dbg(xor_dev->dmadev.dev, "%s %d\n", __func__, __LINE__);  in mv_xor_v2_tasklet()
561 	num_of_pending = mv_xor_v2_get_pending_params(xor_dev, &pending_ptr);  in mv_xor_v2_tasklet()
566 			xor_dev->hw_desq_virt + pending_ptr;  in mv_xor_v2_tasklet()
570 			&xor_dev->sw_desq[next_pending_hw_desc->desc_id];  in mv_xor_v2_tasklet()
589 		spin_lock_bh(&xor_dev->lock);  in mv_xor_v2_tasklet()
593 			 &xor_dev->free_sw_desc);  in mv_xor_v2_tasklet()
596 		spin_unlock_bh(&xor_dev->lock);  in mv_xor_v2_tasklet()
606 		mv_xor_v2_free_desc_from_desq(xor_dev, num_of_pending);  in mv_xor_v2_tasklet()
615 	struct mv_xor_v2_device *xor_dev = dev_get_drvdata(desc->dev);  in mv_xor_v2_set_msi_msg()  local
618 	       xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BALR_OFF);  in mv_xor_v2_set_msi_msg()
620 	       xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_BAHR_OFF);  in mv_xor_v2_set_msi_msg()
622 	       xor_dev->dma_base + MV_XOR_V2_DMA_IMSG_CDAT_OFF);  in mv_xor_v2_set_msi_msg()
625 static int mv_xor_v2_descq_init(struct mv_xor_v2_device *xor_dev)  in mv_xor_v2_descq_init()  argument
631 	       xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_SIZE_OFF);  in mv_xor_v2_descq_init()
634 	writel(lower_32_bits(xor_dev->hw_desq),  in mv_xor_v2_descq_init()
635 	       xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BALR_OFF);  in mv_xor_v2_descq_init()
636 	writel(upper_32_bits(xor_dev->hw_desq),  in mv_xor_v2_descq_init()
637 	       xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_BAHR_OFF);  in mv_xor_v2_descq_init()
648 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);  in mv_xor_v2_descq_init()
652 	writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_ARATTR_OFF);  in mv_xor_v2_descq_init()
654 	reg = readl(xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);  in mv_xor_v2_descq_init()
658 	writel(reg, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_AWATTR_OFF);  in mv_xor_v2_descq_init()
675 	writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_BW_CTRL);  in mv_xor_v2_descq_init()
678 	reg = readl(xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);  in mv_xor_v2_descq_init()
680 	writel(reg, xor_dev->glob_base + MV_XOR_V2_GLOB_PAUSE);  in mv_xor_v2_descq_init()
683 	writel(0, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);  in mv_xor_v2_descq_init()
690 	struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);  in mv_xor_v2_suspend()  local
693 	writel(0x1, xor_dev->dma_base + MV_XOR_V2_DMA_DESQ_STOP_OFF);  in mv_xor_v2_suspend()
700 	struct mv_xor_v2_device *xor_dev = platform_get_drvdata(dev);  in mv_xor_v2_resume()  local
702 	mv_xor_v2_set_desc_size(xor_dev);  in mv_xor_v2_resume()
703 	mv_xor_v2_enable_imsg_thrd(xor_dev);  in mv_xor_v2_resume()
704 	mv_xor_v2_descq_init(xor_dev);  in mv_xor_v2_resume()
711 	struct mv_xor_v2_device *xor_dev;  in mv_xor_v2_probe()  local
721 	xor_dev = devm_kzalloc(&pdev->dev, sizeof(*xor_dev), GFP_KERNEL);  in mv_xor_v2_probe()
722 	if (!xor_dev)  in mv_xor_v2_probe()
726 	xor_dev->dma_base = devm_ioremap_resource(&pdev->dev, res);  in mv_xor_v2_probe()
727 	if (IS_ERR(xor_dev->dma_base))  in mv_xor_v2_probe()
728 		return PTR_ERR(xor_dev->dma_base);  in mv_xor_v2_probe()
731 	xor_dev->glob_base = devm_ioremap_resource(&pdev->dev, res);  in mv_xor_v2_probe()
732 	if (IS_ERR(xor_dev->glob_base))  in mv_xor_v2_probe()
733 		return PTR_ERR(xor_dev->glob_base);  in mv_xor_v2_probe()
735 	platform_set_drvdata(pdev, xor_dev);  in mv_xor_v2_probe()
741 	xor_dev->reg_clk = devm_clk_get(&pdev->dev, "reg");  in mv_xor_v2_probe()
742 	if (PTR_ERR(xor_dev->reg_clk) != -ENOENT) {  in mv_xor_v2_probe()
743 		if (!IS_ERR(xor_dev->reg_clk)) {  in mv_xor_v2_probe()
744 			ret = clk_prepare_enable(xor_dev->reg_clk);  in mv_xor_v2_probe()
748 			return PTR_ERR(xor_dev->reg_clk);  in mv_xor_v2_probe()
752 	xor_dev->clk = devm_clk_get(&pdev->dev, NULL);  in mv_xor_v2_probe()
753 	if (IS_ERR(xor_dev->clk) && PTR_ERR(xor_dev->clk) == -EPROBE_DEFER) {  in mv_xor_v2_probe()
757 	if (!IS_ERR(xor_dev->clk)) {  in mv_xor_v2_probe()
758 		ret = clk_prepare_enable(xor_dev->clk);  in mv_xor_v2_probe()
771 	xor_dev->msi_desc = msi_desc;  in mv_xor_v2_probe()
775 			       dev_name(&pdev->dev), xor_dev);  in mv_xor_v2_probe()
779 	tasklet_init(&xor_dev->irq_tasklet, mv_xor_v2_tasklet,  in mv_xor_v2_probe()
780 		     (unsigned long) xor_dev);  in mv_xor_v2_probe()
782 	xor_dev->desc_size = mv_xor_v2_set_desc_size(xor_dev);  in mv_xor_v2_probe()
784 	dma_cookie_init(&xor_dev->dmachan);  in mv_xor_v2_probe()
791 	xor_dev->hw_desq_virt =  in mv_xor_v2_probe()
793 				   xor_dev->desc_size * MV_XOR_V2_DESC_NUM,  in mv_xor_v2_probe()
794 				   &xor_dev->hw_desq, GFP_KERNEL);  in mv_xor_v2_probe()
795 	if (!xor_dev->hw_desq_virt) {  in mv_xor_v2_probe()
801 	xor_dev->sw_desq = devm_kcalloc(&pdev->dev,  in mv_xor_v2_probe()
804 	if (!xor_dev->sw_desq) {  in mv_xor_v2_probe()
809 	spin_lock_init(&xor_dev->lock);  in mv_xor_v2_probe()
812 	INIT_LIST_HEAD(&xor_dev->free_sw_desc);  in mv_xor_v2_probe()
817 			xor_dev->sw_desq + i;  in mv_xor_v2_probe()
820 					     &xor_dev->dmachan);  in mv_xor_v2_probe()
825 			 &xor_dev->free_sw_desc);  in mv_xor_v2_probe()
828 	dma_dev = &xor_dev->dmadev;  in mv_xor_v2_probe()
849 	xor_dev->dmachan.device = dma_dev;  in mv_xor_v2_probe()
851 	list_add_tail(&xor_dev->dmachan.device_node,  in mv_xor_v2_probe()
854 	mv_xor_v2_enable_imsg_thrd(xor_dev);  in mv_xor_v2_probe()
856 	mv_xor_v2_descq_init(xor_dev);  in mv_xor_v2_probe()
868 			  xor_dev->desc_size * MV_XOR_V2_DESC_NUM,  in mv_xor_v2_probe()
869 			  xor_dev->hw_desq_virt, xor_dev->hw_desq);  in mv_xor_v2_probe()
873 	clk_disable_unprepare(xor_dev->clk);  in mv_xor_v2_probe()
875 	clk_disable_unprepare(xor_dev->reg_clk);  in mv_xor_v2_probe()
881 	struct mv_xor_v2_device *xor_dev = platform_get_drvdata(pdev);  in mv_xor_v2_remove()  local
883 	dma_async_device_unregister(&xor_dev->dmadev);  in mv_xor_v2_remove()
886 			  xor_dev->desc_size * MV_XOR_V2_DESC_NUM,  in mv_xor_v2_remove()
887 			  xor_dev->hw_desq_virt, xor_dev->hw_desq);  in mv_xor_v2_remove()
889 	devm_free_irq(&pdev->dev, xor_dev->msi_desc->irq, xor_dev);  in mv_xor_v2_remove()
893 	tasklet_kill(&xor_dev->irq_tasklet);  in mv_xor_v2_remove()
895 	clk_disable_unprepare(xor_dev->clk);  in mv_xor_v2_remove()