124 void hidma_ll_free(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_free()  argument
128 	if (tre_ch >= lldev->nr_tres) {  in hidma_ll_free()
129 		dev_err(lldev->dev, "invalid TRE number in free:%d", tre_ch);  in hidma_ll_free()
133 	tre = &lldev->trepool[tre_ch];  in hidma_ll_free()
135 		dev_err(lldev->dev, "trying to free an unused TRE:%d", tre_ch);  in hidma_ll_free()
142 int hidma_ll_request(struct hidma_lldev *lldev, u32 sig, const char *dev_name,  in hidma_ll_request()  argument
149 	if (!tre_ch || !lldev)  in hidma_ll_request()
153 	for (i = 0; i < lldev->nr_tres - 1; i++) {  in hidma_ll_request()
154 		if (atomic_add_unless(&lldev->trepool[i].allocated, 1, 1))  in hidma_ll_request()
158 	if (i == (lldev->nr_tres - 1))  in hidma_ll_request()
161 	tre = &lldev->trepool[i];  in hidma_ll_request()
171 	tre->lldev = lldev;  in hidma_ll_request()
173 	tre_local[HIDMA_TRE_CFG_IDX] = (lldev->chidx & 0xFF) << 8;  in hidma_ll_request()
186 	struct hidma_lldev *lldev = (struct hidma_lldev *)arg;  in hidma_ll_tre_complete()  local
189 	while (kfifo_out(&lldev->handoff_fifo, &tre, 1)) {  in hidma_ll_tre_complete()
196 static int hidma_post_completed(struct hidma_lldev *lldev, u8 err_info,  in hidma_post_completed()  argument
203 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_post_completed()
205 	tre_iterator = lldev->tre_processed_off;  in hidma_post_completed()
206 	tre = lldev->pending_tre_list[tre_iterator / HIDMA_TRE_SIZE];  in hidma_post_completed()
208 		spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_post_completed()
209 		dev_warn(lldev->dev, "tre_index [%d] and tre out of sync\n",  in hidma_post_completed()
213 	lldev->pending_tre_list[tre->tre_index] = NULL;  in hidma_post_completed()
219 	if (atomic_dec_return(&lldev->pending_tre_count) < 0) {  in hidma_post_completed()
220 		dev_warn(lldev->dev, "tre count mismatch on completion");  in hidma_post_completed()
221 		atomic_set(&lldev->pending_tre_count, 0);  in hidma_post_completed()
225 				 lldev->tre_ring_size);  in hidma_post_completed()
226 	lldev->tre_processed_off = tre_iterator;  in hidma_post_completed()
227 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_post_completed()
233 	kfifo_put(&lldev->handoff_fifo, tre);  in hidma_post_completed()
234 	tasklet_schedule(&lldev->task);  in hidma_post_completed()
245 static int hidma_handle_tre_completion(struct hidma_lldev *lldev)  in hidma_handle_tre_completion()  argument
247 	u32 evre_ring_size = lldev->evre_ring_size;  in hidma_handle_tre_completion()
252 	evre_write_off = readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);  in hidma_handle_tre_completion()
253 	evre_iterator = lldev->evre_processed_off;  in hidma_handle_tre_completion()
257 		dev_err(lldev->dev, "HW reports invalid EVRE write offset\n");  in hidma_handle_tre_completion()
266 		u32 *current_evre = lldev->evre_ring + evre_iterator;  in hidma_handle_tre_completion()
275 		if (hidma_post_completed(lldev, err_info, err_code))  in hidma_handle_tre_completion()
287 		    readl_relaxed(lldev->evca + HIDMA_EVCA_WRITE_PTR_REG);  in hidma_handle_tre_completion()
294 		if (!hidma_ll_isenabled(lldev))  in hidma_handle_tre_completion()
299 		u32 evre_read_off = (lldev->evre_processed_off +  in hidma_handle_tre_completion()
302 		writel(evre_read_off, lldev->evca + HIDMA_EVCA_DOORBELL_REG);  in hidma_handle_tre_completion()
305 		lldev->evre_processed_off = evre_read_off;  in hidma_handle_tre_completion()
311 void hidma_cleanup_pending_tre(struct hidma_lldev *lldev, u8 err_info,  in hidma_cleanup_pending_tre()  argument
314 	while (atomic_read(&lldev->pending_tre_count)) {  in hidma_cleanup_pending_tre()
315 		if (hidma_post_completed(lldev, err_info, err_code))  in hidma_cleanup_pending_tre()
320 static int hidma_ll_reset(struct hidma_lldev *lldev)  in hidma_ll_reset()  argument
325 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_reset()
328 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_reset()
334 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_reset()
338 		dev_err(lldev->dev, "transfer channel did not reset\n");  in hidma_ll_reset()
342 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_reset()
345 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_reset()
351 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_reset()
357 	lldev->trch_state = HIDMA_CH_DISABLED;  in hidma_ll_reset()
358 	lldev->evch_state = HIDMA_CH_DISABLED;  in hidma_ll_reset()
394 static void hidma_ll_int_handler_internal(struct hidma_lldev *lldev, int cause)  in hidma_ll_int_handler_internal()  argument
399 		dev_err(lldev->dev, "error 0x%x, disabling...\n",  in hidma_ll_int_handler_internal()
403 		writel(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_int_handler_internal()
406 		hidma_ll_disable(lldev);  in hidma_ll_int_handler_internal()
409 		hidma_cleanup_pending_tre(lldev, 0xFF,  in hidma_ll_int_handler_internal()
415 	spin_lock_irqsave(&lldev->lock, irqflags);  in hidma_ll_int_handler_internal()
416 	writel_relaxed(cause, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_int_handler_internal()
417 	spin_unlock_irqrestore(&lldev->lock, irqflags);  in hidma_ll_int_handler_internal()
429 	hidma_handle_tre_completion(lldev);  in hidma_ll_int_handler_internal()
434 	struct hidma_lldev *lldev = arg;  in hidma_ll_inthandler()  local
439 	status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_inthandler()
440 	enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_inthandler()
444 		hidma_ll_int_handler_internal(lldev, cause);  in hidma_ll_inthandler()
450 		status = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_inthandler()
451 		enable = readl_relaxed(lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_inthandler()
460 	struct hidma_lldev *lldev = arg;  in hidma_ll_inthandler_msi()  local
462 	hidma_ll_int_handler_internal(lldev, cause);  in hidma_ll_inthandler_msi()
466 int hidma_ll_enable(struct hidma_lldev *lldev)  in hidma_ll_enable()  argument
471 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_enable()
474 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_enable()
476 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_enable()
480 		dev_err(lldev->dev, "event channel did not get enabled\n");  in hidma_ll_enable()
484 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_enable()
487 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_enable()
489 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_enable()
493 		dev_err(lldev->dev, "transfer channel did not get enabled\n");  in hidma_ll_enable()
497 	lldev->trch_state = HIDMA_CH_ENABLED;  in hidma_ll_enable()
498 	lldev->evch_state = HIDMA_CH_ENABLED;  in hidma_ll_enable()
501 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_enable()
506 void hidma_ll_start(struct hidma_lldev *lldev)  in hidma_ll_start()  argument
510 	spin_lock_irqsave(&lldev->lock, irqflags);  in hidma_ll_start()
511 	writel(lldev->tre_write_offset, lldev->trca + HIDMA_TRCA_DOORBELL_REG);  in hidma_ll_start()
512 	spin_unlock_irqrestore(&lldev->lock, irqflags);  in hidma_ll_start()
515 bool hidma_ll_isenabled(struct hidma_lldev *lldev)  in hidma_ll_isenabled()  argument
519 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_isenabled()
520 	lldev->trch_state = HIDMA_CH_STATE(val);  in hidma_ll_isenabled()
521 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_isenabled()
522 	lldev->evch_state = HIDMA_CH_STATE(val);  in hidma_ll_isenabled()
525 	if (hidma_is_chan_enabled(lldev->trch_state) &&  in hidma_ll_isenabled()
526 	    hidma_is_chan_enabled(lldev->evch_state))  in hidma_ll_isenabled()
532 void hidma_ll_queue_request(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_queue_request()  argument
537 	tre = &lldev->trepool[tre_ch];  in hidma_ll_queue_request()
540 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_ll_queue_request()
541 	tre->tre_index = lldev->tre_write_offset / HIDMA_TRE_SIZE;  in hidma_ll_queue_request()
542 	lldev->pending_tre_list[tre->tre_index] = tre;  in hidma_ll_queue_request()
543 	memcpy(lldev->tre_ring + lldev->tre_write_offset,  in hidma_ll_queue_request()
548 	atomic_inc(&lldev->pending_tre_count);  in hidma_ll_queue_request()
549 	lldev->tre_write_offset = (lldev->tre_write_offset + HIDMA_TRE_SIZE)  in hidma_ll_queue_request()
550 					% lldev->tre_ring_size;  in hidma_ll_queue_request()
551 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_ll_queue_request()
559 int hidma_ll_disable(struct hidma_lldev *lldev)  in hidma_ll_disable()  argument
565 	if (!hidma_ll_isenabled(lldev))  in hidma_ll_disable()
568 	val = readl(lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_disable()
571 	writel(val, lldev->trca + HIDMA_TRCA_CTRLSTS_REG);  in hidma_ll_disable()
577 	ret = readl_poll_timeout(lldev->trca + HIDMA_TRCA_CTRLSTS_REG, val,  in hidma_ll_disable()
583 	val = readl(lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_disable()
586 	writel(val, lldev->evca + HIDMA_EVCA_CTRLSTS_REG);  in hidma_ll_disable()
592 	ret = readl_poll_timeout(lldev->evca + HIDMA_EVCA_CTRLSTS_REG, val,  in hidma_ll_disable()
598 	lldev->trch_state = HIDMA_CH_SUSPENDED;  in hidma_ll_disable()
599 	lldev->evch_state = HIDMA_CH_SUSPENDED;  in hidma_ll_disable()
602 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_disable()
606 void hidma_ll_set_transfer_params(struct hidma_lldev *lldev, u32 tre_ch,  in hidma_ll_set_transfer_params()  argument
613 	if (tre_ch >= lldev->nr_tres) {  in hidma_ll_set_transfer_params()
614 		dev_err(lldev->dev, "invalid TRE number in transfer params:%d",  in hidma_ll_set_transfer_params()
619 	tre = &lldev->trepool[tre_ch];  in hidma_ll_set_transfer_params()
621 		dev_err(lldev->dev, "trying to set params on an unused TRE:%d",  in hidma_ll_set_transfer_params()
641 int hidma_ll_setup(struct hidma_lldev *lldev)  in hidma_ll_setup()  argument
646 	u32 nr_tres = lldev->nr_tres;  in hidma_ll_setup()
648 	atomic_set(&lldev->pending_tre_count, 0);  in hidma_ll_setup()
649 	lldev->tre_processed_off = 0;  in hidma_ll_setup()
650 	lldev->evre_processed_off = 0;  in hidma_ll_setup()
651 	lldev->tre_write_offset = 0;  in hidma_ll_setup()
654 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup()
657 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_setup()
658 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup()
660 	rc = hidma_ll_reset(lldev);  in hidma_ll_setup()
668 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_setup()
669 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup()
672 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup()
674 	addr = lldev->tre_dma;  in hidma_ll_setup()
675 	writel(lower_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_LOW_REG);  in hidma_ll_setup()
676 	writel(upper_32_bits(addr), lldev->trca + HIDMA_TRCA_RING_HIGH_REG);  in hidma_ll_setup()
677 	writel(lldev->tre_ring_size, lldev->trca + HIDMA_TRCA_RING_LEN_REG);  in hidma_ll_setup()
679 	addr = lldev->evre_dma;  in hidma_ll_setup()
680 	writel(lower_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_LOW_REG);  in hidma_ll_setup()
681 	writel(upper_32_bits(addr), lldev->evca + HIDMA_EVCA_RING_HIGH_REG);  in hidma_ll_setup()
683 			lldev->evca + HIDMA_EVCA_RING_LEN_REG);  in hidma_ll_setup()
686 	hidma_ll_setup_irq(lldev, lldev->msi_support);  in hidma_ll_setup()
688 	rc = hidma_ll_enable(lldev);  in hidma_ll_setup()
695 void hidma_ll_setup_irq(struct hidma_lldev *lldev, bool msi)  in hidma_ll_setup_irq()  argument
699 	lldev->msi_support = msi;  in hidma_ll_setup_irq()
702 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup_irq()
703 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup_irq()
706 	val = readl(lldev->evca + HIDMA_EVCA_INTCTRL_REG);  in hidma_ll_setup_irq()
708 	if (!lldev->msi_support)  in hidma_ll_setup_irq()
710 	writel(val, lldev->evca + HIDMA_EVCA_INTCTRL_REG);  in hidma_ll_setup_irq()
713 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_setup_irq()
714 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_setup_irq()
722 	struct hidma_lldev *lldev;  in hidma_ll_init()  local
736 	lldev = devm_kzalloc(dev, sizeof(struct hidma_lldev), GFP_KERNEL);  in hidma_ll_init()
737 	if (!lldev)  in hidma_ll_init()
740 	lldev->evca = evca;  in hidma_ll_init()
741 	lldev->trca = trca;  in hidma_ll_init()
742 	lldev->dev = dev;  in hidma_ll_init()
744 	lldev->trepool = devm_kcalloc(lldev->dev, nr_tres, sz, GFP_KERNEL);  in hidma_ll_init()
745 	if (!lldev->trepool)  in hidma_ll_init()
748 	required_bytes = sizeof(lldev->pending_tre_list[0]);  in hidma_ll_init()
749 	lldev->pending_tre_list = devm_kcalloc(dev, nr_tres, required_bytes,  in hidma_ll_init()
751 	if (!lldev->pending_tre_list)  in hidma_ll_init()
755 	lldev->tre_ring = dmam_alloc_coherent(dev, sz, &lldev->tre_dma,  in hidma_ll_init()
757 	if (!lldev->tre_ring)  in hidma_ll_init()
760 	memset(lldev->tre_ring, 0, (HIDMA_TRE_SIZE + 1) * nr_tres);  in hidma_ll_init()
761 	lldev->tre_ring_size = HIDMA_TRE_SIZE * nr_tres;  in hidma_ll_init()
762 	lldev->nr_tres = nr_tres;  in hidma_ll_init()
765 	if (!IS_ALIGNED(lldev->tre_dma, HIDMA_TRE_SIZE)) {  in hidma_ll_init()
768 		tre_ring_shift = lldev->tre_dma % HIDMA_TRE_SIZE;  in hidma_ll_init()
770 		lldev->tre_dma += tre_ring_shift;  in hidma_ll_init()
771 		lldev->tre_ring += tre_ring_shift;  in hidma_ll_init()
775 	lldev->evre_ring = dmam_alloc_coherent(dev, sz, &lldev->evre_dma,  in hidma_ll_init()
777 	if (!lldev->evre_ring)  in hidma_ll_init()
780 	memset(lldev->evre_ring, 0, (HIDMA_EVRE_SIZE + 1) * nr_tres);  in hidma_ll_init()
781 	lldev->evre_ring_size = HIDMA_EVRE_SIZE * nr_tres;  in hidma_ll_init()
784 	if (!IS_ALIGNED(lldev->evre_dma, HIDMA_EVRE_SIZE)) {  in hidma_ll_init()
787 		evre_ring_shift = lldev->evre_dma % HIDMA_EVRE_SIZE;  in hidma_ll_init()
789 		lldev->evre_dma += evre_ring_shift;  in hidma_ll_init()
790 		lldev->evre_ring += evre_ring_shift;  in hidma_ll_init()
792 	lldev->nr_tres = nr_tres;  in hidma_ll_init()
793 	lldev->chidx = chidx;  in hidma_ll_init()
796 	rc = kfifo_alloc(&lldev->handoff_fifo, sz, GFP_KERNEL);  in hidma_ll_init()
800 	rc = hidma_ll_setup(lldev);  in hidma_ll_init()
804 	spin_lock_init(&lldev->lock);  in hidma_ll_init()
805 	tasklet_init(&lldev->task, hidma_ll_tre_complete, (unsigned long)lldev);  in hidma_ll_init()
806 	lldev->initialized = 1;  in hidma_ll_init()
807 	writel(ENABLE_IRQS, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_init()
808 	return lldev;  in hidma_ll_init()
811 int hidma_ll_uninit(struct hidma_lldev *lldev)  in hidma_ll_uninit()  argument
817 	if (!lldev)  in hidma_ll_uninit()
820 	if (!lldev->initialized)  in hidma_ll_uninit()
823 	lldev->initialized = 0;  in hidma_ll_uninit()
825 	required_bytes = sizeof(struct hidma_tre) * lldev->nr_tres;  in hidma_ll_uninit()
826 	tasklet_kill(&lldev->task);  in hidma_ll_uninit()
827 	memset(lldev->trepool, 0, required_bytes);  in hidma_ll_uninit()
828 	lldev->trepool = NULL;  in hidma_ll_uninit()
829 	atomic_set(&lldev->pending_tre_count, 0);  in hidma_ll_uninit()
830 	lldev->tre_write_offset = 0;  in hidma_ll_uninit()
832 	rc = hidma_ll_reset(lldev);  in hidma_ll_uninit()
838 	val = readl(lldev->evca + HIDMA_EVCA_IRQ_STAT_REG);  in hidma_ll_uninit()
839 	writel(val, lldev->evca + HIDMA_EVCA_IRQ_CLR_REG);  in hidma_ll_uninit()
840 	writel(0, lldev->evca + HIDMA_EVCA_IRQ_EN_REG);  in hidma_ll_uninit()
844 enum dma_status hidma_ll_status(struct hidma_lldev *lldev, u32 tre_ch)  in hidma_ll_status()  argument
851 	spin_lock_irqsave(&lldev->lock, flags);  in hidma_ll_status()
853 	tre = &lldev->trepool[tre_ch];  in hidma_ll_status()
862 	spin_unlock_irqrestore(&lldev->lock, flags);  in hidma_ll_status()