344 #define IS_BUSY(chan)	(CIRC_SPACE(bchan->tail, bchan->head,\
425 static void bam_reset_channel(struct bam_chan *bchan)  in bam_reset_channel()  argument
427 	struct bam_device *bdev = bchan->bdev;  in bam_reset_channel()
429 	lockdep_assert_held(&bchan->vc.lock);  in bam_reset_channel()
432 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
433 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_RST));  in bam_reset_channel()
439 	bchan->initialized = 0;  in bam_reset_channel()
449 static void bam_chan_init_hw(struct bam_chan *bchan,  in bam_chan_init_hw()  argument
452 	struct bam_device *bdev = bchan->bdev;  in bam_chan_init_hw()
456 	bam_reset_channel(bchan);  in bam_chan_init_hw()
462 	writel_relaxed(ALIGN(bchan->fifo_phys, sizeof(struct bam_desc_hw)),  in bam_chan_init_hw()
463 			bam_addr(bdev, bchan->id, BAM_P_DESC_FIFO_ADDR));  in bam_chan_init_hw()
465 			bam_addr(bdev, bchan->id, BAM_P_FIFO_SIZES));  in bam_chan_init_hw()
469 			bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_chan_init_hw()
473 	val |= BIT(bchan->id);  in bam_chan_init_hw()
484 	writel_relaxed(val, bam_addr(bdev, bchan->id, BAM_P_CTRL));  in bam_chan_init_hw()
486 	bchan->initialized = 1;  in bam_chan_init_hw()
489 	bchan->head = 0;  in bam_chan_init_hw()
490 	bchan->tail = 0;  in bam_chan_init_hw()
501 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_alloc_chan()  local
502 	struct bam_device *bdev = bchan->bdev;  in bam_alloc_chan()
504 	if (bchan->fifo_virt)  in bam_alloc_chan()
508 	bchan->fifo_virt = dma_alloc_wc(bdev->dev, BAM_DESC_FIFO_SIZE,  in bam_alloc_chan()
509 					&bchan->fifo_phys, GFP_KERNEL);  in bam_alloc_chan()
511 	if (!bchan->fifo_virt) {  in bam_alloc_chan()
536 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_free_chan()  local
537 	struct bam_device *bdev = bchan->bdev;  in bam_free_chan()
548 	if (!list_empty(&bchan->desc_list)) {  in bam_free_chan()
549 		dev_err(bchan->bdev->dev, "Cannot free busy channel\n");  in bam_free_chan()
553 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_free_chan()
554 	bam_reset_channel(bchan);  in bam_free_chan()
555 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_free_chan()
557 	dma_free_wc(bdev->dev, BAM_DESC_FIFO_SIZE, bchan->fifo_virt,  in bam_free_chan()
558 		    bchan->fifo_phys);  in bam_free_chan()
559 	bchan->fifo_virt = NULL;  in bam_free_chan()
563 	val &= ~BIT(bchan->id);  in bam_free_chan()
567 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_IRQ_EN));  in bam_free_chan()
585 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_slave_config()  local
588 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_slave_config()
589 	memcpy(&bchan->slave, cfg, sizeof(*cfg));  in bam_slave_config()
590 	bchan->reconfigure = 1;  in bam_slave_config()
591 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_slave_config()
611 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_prep_slave_sg()  local
612 	struct bam_device *bdev = bchan->bdev;  in bam_prep_slave_sg()
673 	return vchan_tx_prep(&bchan->vc, &async_desc->vd, flags);  in bam_prep_slave_sg()
690 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_dma_terminate_all()  local
696 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_dma_terminate_all()
710 	if (!list_empty(&bchan->desc_list)) {  in bam_dma_terminate_all()
711 		async_desc = list_first_entry(&bchan->desc_list,  in bam_dma_terminate_all()
713 		bam_chan_init_hw(bchan, async_desc->dir);  in bam_dma_terminate_all()
717 				 &bchan->desc_list, desc_node) {  in bam_dma_terminate_all()
718 		list_add(&async_desc->vd.node, &bchan->vc.desc_issued);  in bam_dma_terminate_all()
722 	vchan_get_all_descriptors(&bchan->vc, &head);  in bam_dma_terminate_all()
723 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_dma_terminate_all()
725 	vchan_dma_desc_free_list(&bchan->vc, &head);  in bam_dma_terminate_all()
737 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_pause()  local
738 	struct bam_device *bdev = bchan->bdev;  in bam_pause()
746 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_pause()
747 	writel_relaxed(1, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_pause()
748 	bchan->paused = 1;  in bam_pause()
749 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_pause()
763 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_resume()  local
764 	struct bam_device *bdev = bchan->bdev;  in bam_resume()
772 	spin_lock_irqsave(&bchan->vc.lock, flag);  in bam_resume()
773 	writel_relaxed(0, bam_addr(bdev, bchan->id, BAM_P_HALT));  in bam_resume()
774 	bchan->paused = 0;  in bam_resume()
775 	spin_unlock_irqrestore(&bchan->vc.lock, flag);  in bam_resume()
802 		struct bam_chan *bchan = &bdev->channels[i];  in process_channel_irqs()  local
812 		spin_lock_irqsave(&bchan->vc.lock, flags);  in process_channel_irqs()
819 		avail = CIRC_CNT(offset, bchan->head, MAX_DESCRIPTORS + 1);  in process_channel_irqs()
821 		if (offset < bchan->head)  in process_channel_irqs()
825 					 &bchan->desc_list, desc_node) {  in process_channel_irqs()
831 			bchan->head += async_desc->xfer_len;  in process_channel_irqs()
832 			bchan->head %= MAX_DESCRIPTORS;  in process_channel_irqs()
847 					 &bchan->vc.desc_issued);  in process_channel_irqs()
852 		spin_unlock_irqrestore(&bchan->vc.lock, flags);  in process_channel_irqs()
910 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_tx_status()  local
923 		return bchan->paused ? DMA_PAUSED : ret;  in bam_tx_status()
925 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_tx_status()
926 	vd = vchan_find_desc(&bchan->vc, cookie);  in bam_tx_status()
930 		list_for_each_entry(async_desc, &bchan->desc_list, desc_node) {  in bam_tx_status()
940 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_tx_status()
944 	if (ret == DMA_IN_PROGRESS && bchan->paused)  in bam_tx_status()
955 static void bam_apply_new_config(struct bam_chan *bchan,  in bam_apply_new_config()  argument
958 	struct bam_device *bdev = bchan->bdev;  in bam_apply_new_config()
963 			maxburst = bchan->slave.src_maxburst;  in bam_apply_new_config()
965 			maxburst = bchan->slave.dst_maxburst;  in bam_apply_new_config()
971 	bchan->reconfigure = 0;  in bam_apply_new_config()
978 static void bam_start_dma(struct bam_chan *bchan)  in bam_start_dma()  argument
980 	struct virt_dma_desc *vd = vchan_next_desc(&bchan->vc);  in bam_start_dma()
981 	struct bam_device *bdev = bchan->bdev;  in bam_start_dma()
984 	struct bam_desc_hw *fifo = PTR_ALIGN(bchan->fifo_virt,  in bam_start_dma()
990 	lockdep_assert_held(&bchan->vc.lock);  in bam_start_dma()
999 	while (vd && !IS_BUSY(bchan)) {  in bam_start_dma()
1005 		if (!bchan->initialized)  in bam_start_dma()
1006 			bam_chan_init_hw(bchan, async_desc->dir);  in bam_start_dma()
1009 		if (bchan->reconfigure)  in bam_start_dma()
1010 			bam_apply_new_config(bchan, async_desc->dir);  in bam_start_dma()
1013 		avail = CIRC_SPACE(bchan->tail, bchan->head,  in bam_start_dma()
1026 		vd = vchan_next_desc(&bchan->vc);  in bam_start_dma()
1044 		if (bchan->tail + async_desc->xfer_len > MAX_DESCRIPTORS) {  in bam_start_dma()
1045 			u32 partial = MAX_DESCRIPTORS - bchan->tail;  in bam_start_dma()
1047 			memcpy(&fifo[bchan->tail], desc,  in bam_start_dma()
1053 			memcpy(&fifo[bchan->tail], desc,  in bam_start_dma()
1058 		bchan->tail += async_desc->xfer_len;  in bam_start_dma()
1059 		bchan->tail %= MAX_DESCRIPTORS;  in bam_start_dma()
1060 		list_add_tail(&async_desc->desc_node, &bchan->desc_list);  in bam_start_dma()
1065 	writel_relaxed(bchan->tail * sizeof(struct bam_desc_hw),  in bam_start_dma()
1066 			bam_addr(bdev, bchan->id, BAM_P_EVNT_REG));  in bam_start_dma()
1081 	struct bam_chan *bchan;  in dma_tasklet()  local
1087 		bchan = &bdev->channels[i];  in dma_tasklet()
1088 		spin_lock_irqsave(&bchan->vc.lock, flags);  in dma_tasklet()
1090 		if (!list_empty(&bchan->vc.desc_issued) && !IS_BUSY(bchan))  in dma_tasklet()
1091 			bam_start_dma(bchan);  in dma_tasklet()
1092 		spin_unlock_irqrestore(&bchan->vc.lock, flags);  in dma_tasklet()
1105 	struct bam_chan *bchan = to_bam_chan(chan);  in bam_issue_pending()  local
1108 	spin_lock_irqsave(&bchan->vc.lock, flags);  in bam_issue_pending()
1111 	if (vchan_issue_pending(&bchan->vc) && !IS_BUSY(bchan))  in bam_issue_pending()
1112 		bam_start_dma(bchan);  in bam_issue_pending()
1114 	spin_unlock_irqrestore(&bchan->vc.lock, flags);  in bam_issue_pending()
1207 static void bam_channel_init(struct bam_device *bdev, struct bam_chan *bchan,  in bam_channel_init()  argument
1210 	bchan->id = index;  in bam_channel_init()
1211 	bchan->bdev = bdev;  in bam_channel_init()
1213 	vchan_init(&bchan->vc, &bdev->common);  in bam_channel_init()
1214 	bchan->vc.desc_free = bam_dma_free_desc;  in bam_channel_init()
1215 	INIT_LIST_HEAD(&bchan->desc_list);  in bam_channel_init()