116 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_free()  argument
120 	if (tre_ch >= lldev->nr_tres) {  in hidma_ll_free()
121 		dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);  in hidma_ll_free()
125 	tre = &lldev->trepool[tre_ch];  in hidma_ll_free()
127 		dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);  in hidma_ll_free()
134 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,  in hidma_ll_request()  argument
141 	if (!tre_ch || !lldev)  in hidma_ll_request()
145 	for (i = 0; i < lldev->nr_tres - 1; i++) {  in hidma_ll_request()
146 		if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))  in hidma_ll_request()
150 	if (i == (lldev->nr_tres - 1))  in hidma_ll_request()
153 	tre = &lldev->trepool[i];  in hidma_ll_request()
163 	tre->lldev = lldev;  in hidma_ll_request()
165 	tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;  in hidma_ll_request()
178 	struct hidma_lldev *lldev = from_tasklet(lldev, t, task);  in hidma_ll_tre_complete()  local
181 	while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {  in hidma_ll_tre_complete()
188 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,  in hidma_post_completed()  argument
195 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_post_completed()
197 	tre_iterator = lldev->tre_processed_off;  in hidma_post_completed()
198 	tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];  in hidma_post_completed()
200 		spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_post_completed()
201 		dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",  in hidma_post_completed()
205 	lldev->pending_tre_list[tre->tre_index] = NULL;  in hidma_post_completed()
211 	if (atomic_dec_return(&lldev->pending_tre_count) < 0) {  in hidma_post_completed()
212 		dev_warn(lldev->dev, "tre count mismatch on completion");  in hidma_post_completed()
213 		atomic_set(&lldev->pending_tre_count, 0);  in hidma_post_completed()
217 				 lldev->tre_ring_size);  in hidma_post_completed()
218 	lldev->tre_processed_off = tre_iterator;  in hidma_post_completed()
219 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_post_completed()
225 	kfifo_put(&lldev->handoff_fifo, tre);  in hidma_post_completed()
226 	tasklet_schedule(&lldev->task);  in hidma_post_completed()
237 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)  in hidma_handle_tre_completion()  argument
239 	u32 evre_ring_size = lldev->evre_ring_size;  in hidma_handle_tre_completion()
244 	evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);  in hidma_handle_tre_completion()
245 	evre_iterator = lldev->evre_processed_off;  in hidma_handle_tre_completion()
249 		dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");  in hidma_handle_tre_completion()
258 		u32 *current_evre = lldev->evre_ring + evre_iterator;  in hidma_handle_tre_completion()
267 		if (hidma_post_completed(lldev, err_info, err_code))  in hidma_handle_tre_completion()
279 		    readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);  in hidma_handle_tre_completion()
286 		if (!hidma_ll_isenabled(lldev))  in hidma_handle_tre_completion()
291 		u32 evre_read_off = (lldev->evre_processed_off +  in hidma_handle_tre_completion()
294 		writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);  in hidma_handle_tre_completion()
297 		lldev->evre_processed_off = evre_read_off;  in hidma_handle_tre_completion()
303 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,  in hidma_cleanup_pending_tre()  argument
306 	while (atomic_read(&lldev->pending_tre_count)) {  in hidma_cleanup_pending_tre()
307 		if (hidma_post_completed(lldev, err_info, err_code))  in hidma_cleanup_pending_tre()
312 static int hidma_ll_reset(struct hidma_lldev *lldev)  in hidma_ll_reset()  argument
317 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_reset()
320 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_reset()
326 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_reset()
330 		dev_err(lldev->dev, "transfer channel did not reset\n");  in hidma_ll_reset()
334 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_reset()
337 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_reset()
343 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_reset()
349 	lldev->trch_state = HIDMA_CH_DISABLED;  in hidma_ll_reset()
350 	lldev->evch_state = HIDMA_CH_DISABLED;  in hidma_ll_reset()
386 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)  in hidma_ll_int_handler_internal()  argument
391 		dev_err(lldev->dev, "error 0x%x, disabling...\n",  in hidma_ll_int_handler_internal()
395 		writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_int_handler_internal()
398 		hidma_ll_disable(lldev);  in hidma_ll_int_handler_internal()
401 		hidma_cleanup_pending_tre(lldev, 0xFF,  in hidma_ll_int_handler_internal()
407 	spin_lock_irqsave(&lldev->lock, irqflags);  in hidma_ll_int_handler_internal()
408 	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_int_handler_internal()
409 	spin_unlock_irqrestore(&lldev->lock, irqflags);  in hidma_ll_int_handler_internal()
421 	hidma_handle_tre_completion(lldev);  in hidma_ll_int_handler_internal()
426 	struct hidma_lldev *lldev = arg;  in hidma_ll_inthandler()  local
431 	status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_inthandler()
432 	enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_inthandler()
436 		hidma_ll_int_handler_internal(lldev, cause);  in hidma_ll_inthandler()
442 		status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_inthandler()
443 		enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_inthandler()
452 	struct hidma_lldev *lldev = arg;  in hidma_ll_inthandler_msi()  local
454 	hidma_ll_int_handler_internal(lldev, cause);  in hidma_ll_inthandler_msi()
458 int hidma_ll_enable(struct hidma_lldev *lldev)  in hidma_ll_enable()  argument
463 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_enable()
466 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_enable()
468 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_enable()
472 		dev_err(lldev->dev, "event channel did not get enabled\n");  in hidma_ll_enable()
476 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_enable()
479 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_enable()
481 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_enable()
485 		dev_err(lldev->dev, "transfer channel did not get enabled\n");  in hidma_ll_enable()
489 	lldev->trch_state = HIDMA_CH_ENABLED;  in hidma_ll_enable()
490 	lldev->evch_state = HIDMA_CH_ENABLED;  in hidma_ll_enable()
493 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_enable()
498 void hidma_ll_start(struct hidma_lldev *lldev)  in hidma_ll_start()  argument
502 	spin_lock_irqsave(&lldev->lock, irqflags);  in hidma_ll_start()
503 	writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);  in hidma_ll_start()
504 	spin_unlock_irqrestore(&lldev->lock, irqflags);  in hidma_ll_start()
507 bool hidma_ll_isenabled(struct hidma_lldev *lldev)  in hidma_ll_isenabled()  argument
511 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_isenabled()
512 	lldev->trch_state = HIDMA_CH_STATE(val);  in hidma_ll_isenabled()
513 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_isenabled()
514 	lldev->evch_state = HIDMA_CH_STATE(val);  in hidma_ll_isenabled()
517 	if (hidma_is_chan_enabled(lldev->trch_state) &&  in hidma_ll_isenabled()
518 	    hidma_is_chan_enabled(lldev->evch_state))  in hidma_ll_isenabled()
524 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_queue_request()  argument
529 	tre = &lldev->trepool[tre_ch];  in hidma_ll_queue_request()
532 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_ll_queue_request()
533 	tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;  in hidma_ll_queue_request()
534 	lldev->pending_tre_list[tre->tre_index] = tre;  in hidma_ll_queue_request()
535 	memcpy(lldev->tre_ring + lldev->tre_write_offset,  in hidma_ll_queue_request()
540 	atomic_inc(&lldev->pending_tre_count);  in hidma_ll_queue_request()
541 	lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)  in hidma_ll_queue_request()
542 					% lldev->tre_ring_size;  in hidma_ll_queue_request()
543 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_ll_queue_request()
551 int hidma_ll_disable(struct hidma_lldev *lldev)  in hidma_ll_disable()  argument
557 	if (!hidma_ll_isenabled(lldev))  in hidma_ll_disable()
560 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_disable()
563 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_disable()
569 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_disable()
575 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_disable()
578 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_disable()
584 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_disable()
590 	lldev->trch_state = HIDMA_CH_SUSPENDED;  in hidma_ll_disable()
591 	lldev->evch_state = HIDMA_CH_SUSPENDED;  in hidma_ll_disable()
594 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_disable()
598 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,  in hidma_ll_set_transfer_params()  argument
605 	if (tre_ch >= lldev->nr_tres) {  in hidma_ll_set_transfer_params()
606 		dev_err(lldev->dev, "invalid TRE number in transfer params:%d",  in hidma_ll_set_transfer_params()
611 	tre = &lldev->trepool[tre_ch];  in hidma_ll_set_transfer_params()
613 		dev_err(lldev->dev, "trying to set params on an unused TRE:%d",  in hidma_ll_set_transfer_params()
633 int hidma_ll_setup(struct hidma_lldev *lldev)  in hidma_ll_setup()  argument
638 	u32 nr_tres = lldev->nr_tres;  in hidma_ll_setup()
640 	atomic_set(&lldev->pending_tre_count, 0);  in hidma_ll_setup()
641 	lldev->tre_processed_off = 0;  in hidma_ll_setup()
642 	lldev->evre_processed_off = 0;  in hidma_ll_setup()
643 	lldev->tre_write_offset = 0;  in hidma_ll_setup()
646 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup()
649 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_setup()
650 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup()
652 	rc = hidma_ll_reset(lldev);  in hidma_ll_setup()
660 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_setup()
661 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup()
664 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup()
666 	addr = lldev->tre_dma;  in hidma_ll_setup()
667 	writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);  in hidma_ll_setup()
668 	writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);  in hidma_ll_setup()
669 	writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);  in hidma_ll_setup()
671 	addr = lldev->evre_dma;  in hidma_ll_setup()
672 	writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);  in hidma_ll_setup()
673 	writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);  in hidma_ll_setup()
675 			lldev->evca + HIDMA_EVCA_RING_LEN_REG);  in hidma_ll_setup()
678 	hidma_ll_setup_irq(lldev, lldev->msi_support);  in hidma_ll_setup()
680 	rc = hidma_ll_enable(lldev);  in hidma_ll_setup()
687 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)  in hidma_ll_setup_irq()  argument
691 	lldev->msi_support = msi;  in hidma_ll_setup_irq()
694 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup_irq()
695 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup_irq()
698 	val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);  in hidma_ll_setup_irq()
700 	if (!lldev->msi_support)  in hidma_ll_setup_irq()
702 	writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);  in hidma_ll_setup_irq()
705 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup_irq()
706 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup_irq()
714 	struct hidma_lldev *lldev;  in hidma_ll_init()  local
728 	lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);  in hidma_ll_init()
729 	if (!lldev)  in hidma_ll_init()
732 	lldev->evca = evca;  in hidma_ll_init()
733 	lldev->trca = trca;  in hidma_ll_init()
734 	lldev->dev = dev;  in hidma_ll_init()
736 	lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);  in hidma_ll_init()
737 	if (!lldev->trepool)  in hidma_ll_init()
740 	required_bytes = sizeof(lldev->pending_tre_list[0]);  in hidma_ll_init()
741 	lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,  in hidma_ll_init()
743 	if (!lldev->pending_tre_list)  in hidma_ll_init()
747 	lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,  in hidma_ll_init()
749 	if (!lldev->tre_ring)  in hidma_ll_init()
752 	lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;  in hidma_ll_init()
753 	lldev->nr_tres = nr_tres;  in hidma_ll_init()
756 	if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {  in hidma_ll_init()
759 		tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;  in hidma_ll_init()
761 		lldev->tre_dma += tre_ring_shift;  in hidma_ll_init()
762 		lldev->tre_ring += tre_ring_shift;  in hidma_ll_init()
766 	lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,  in hidma_ll_init()
768 	if (!lldev->evre_ring)  in hidma_ll_init()
771 	lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;  in hidma_ll_init()
774 	if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {  in hidma_ll_init()
777 		evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;  in hidma_ll_init()
779 		lldev->evre_dma += evre_ring_shift;  in hidma_ll_init()
780 		lldev->evre_ring += evre_ring_shift;  in hidma_ll_init()
782 	lldev->nr_tres = nr_tres;  in hidma_ll_init()
783 	lldev->chidx = chidx;  in hidma_ll_init()
786 	rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);  in hidma_ll_init()
790 	rc = hidma_ll_setup(lldev);  in hidma_ll_init()
794 	spin_lock_init(&lldev->lock);  in hidma_ll_init()
795 	tasklet_setup(&lldev->task, hidma_ll_tre_complete);  in hidma_ll_init()
796 	lldev->initialized = 1;  in hidma_ll_init()
797 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_init()
798 	return lldev;  in hidma_ll_init()
801 int hidma_ll_uninit(struct hidma_lldev *lldev)  in hidma_ll_uninit()  argument
807 	if (!lldev)  in hidma_ll_uninit()
810 	if (!lldev->initialized)  in hidma_ll_uninit()
813 	lldev->initialized = 0;  in hidma_ll_uninit()
815 	required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;  in hidma_ll_uninit()
816 	tasklet_kill(&lldev->task);  in hidma_ll_uninit()
817 	memset(lldev->trepool, 0, required_bytes);  in hidma_ll_uninit()
818 	lldev->trepool = NULL;  in hidma_ll_uninit()
819 	atomic_set(&lldev->pending_tre_count, 0);  in hidma_ll_uninit()
820 	lldev->tre_write_offset = 0;  in hidma_ll_uninit()
822 	rc = hidma_ll_reset(lldev);  in hidma_ll_uninit()
828 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_uninit()
829 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_uninit()
830 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_uninit()
834 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_status()  argument
841 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_ll_status()
843 	tre = &lldev->trepool[tre_ch];  in hidma_ll_status()
852 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_ll_status()