59 static void idma64_chan_init(struct idma64 *idma64, struct idma64_chan *idma64c)  in idma64_chan_init()  argument
67 	channel_writel(idma64c, CFG_LO, cfglo);  in idma64_chan_init()
68 	channel_writel(idma64c, CFG_HI, cfghi);  in idma64_chan_init()
71 	channel_set_bit(idma64, MASK(XFER), idma64c->mask);  in idma64_chan_init()
72 	channel_set_bit(idma64, MASK(ERROR), idma64c->mask);  in idma64_chan_init()
84 static void idma64_chan_stop(struct idma64 *idma64, struct idma64_chan *idma64c)  in idma64_chan_stop()  argument
86 	channel_clear_bit(idma64, CH_EN, idma64c->mask);  in idma64_chan_stop()
89 static void idma64_chan_start(struct idma64 *idma64, struct idma64_chan *idma64c)  in idma64_chan_start()  argument
91 	struct idma64_desc *desc = idma64c->desc;  in idma64_chan_start()
94 	channel_writeq(idma64c, SAR, 0);  in idma64_chan_start()
95 	channel_writeq(idma64c, DAR, 0);  in idma64_chan_start()
97 	channel_writel(idma64c, CTL_HI, IDMA64C_CTLH_BLOCK_TS(~0UL));  in idma64_chan_start()
98 	channel_writel(idma64c, CTL_LO, IDMA64C_CTLL_LLP_S_EN | IDMA64C_CTLL_LLP_D_EN);  in idma64_chan_start()
100 	channel_writeq(idma64c, LLP, hw->llp);  in idma64_chan_start()
102 	channel_set_bit(idma64, CH_EN, idma64c->mask);  in idma64_chan_start()
105 static void idma64_stop_transfer(struct idma64_chan *idma64c)  in idma64_stop_transfer()  argument
107 	struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device);  in idma64_stop_transfer()
109 	idma64_chan_stop(idma64, idma64c);  in idma64_stop_transfer()
112 static void idma64_start_transfer(struct idma64_chan *idma64c)  in idma64_start_transfer()  argument
114 	struct idma64 *idma64 = to_idma64(idma64c->vchan.chan.device);  in idma64_start_transfer()
118 	vdesc = vchan_next_desc(&idma64c->vchan);  in idma64_start_transfer()
120 		idma64c->desc = NULL;  in idma64_start_transfer()
125 	idma64c->desc = to_idma64_desc(vdesc);  in idma64_start_transfer()
128 	idma64_chan_init(idma64, idma64c);  in idma64_start_transfer()
131 	idma64_chan_start(idma64, idma64c);  in idma64_start_transfer()
139 	struct idma64_chan *idma64c = &idma64->chan[c];  in idma64_chan_irq()  local
143 	stat = this_cpu_ptr(idma64c->vchan.chan.local);  in idma64_chan_irq()
145 	spin_lock(&idma64c->vchan.lock);  in idma64_chan_irq()
146 	desc = idma64c->desc;  in idma64_chan_irq()
149 			dma_writel(idma64, CLEAR(ERROR), idma64c->mask);  in idma64_chan_irq()
152 			dma_writel(idma64, CLEAR(XFER), idma64c->mask);  in idma64_chan_irq()
156 			idma64_start_transfer(idma64c);  in idma64_chan_irq()
160 		if (idma64c->desc == NULL || desc->status == DMA_ERROR)  in idma64_chan_irq()
161 			idma64_stop_transfer(idma64c);  in idma64_chan_irq()
163 	spin_unlock(&idma64c->vchan.lock);  in idma64_chan_irq()
208 static void idma64_desc_free(struct idma64_chan *idma64c,  in idma64_desc_free()  argument
218 			dma_pool_free(idma64c->pool, hw->lli, hw->llp);  in idma64_desc_free()
228 	struct idma64_chan *idma64c = to_idma64_chan(vdesc->tx.chan);  in idma64_vdesc_free()  local
230 	idma64_desc_free(idma64c, to_idma64_desc(vdesc));  in idma64_vdesc_free()
272 static void idma64_desc_fill(struct idma64_chan *idma64c,  in idma64_desc_fill()  argument
275 	struct dma_slave_config *config = &idma64c->config;  in idma64_desc_fill()
301 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_prep_slave_sg()  local
314 		hw->lli = dma_pool_alloc(idma64c->pool, GFP_NOWAIT, &hw->llp);  in idma64_prep_slave_sg()
317 			idma64_desc_free(idma64c, desc);  in idma64_prep_slave_sg()
329 	idma64_desc_fill(idma64c, desc);  in idma64_prep_slave_sg()
330 	return vchan_tx_prep(&idma64c->vchan, &desc->vdesc, flags);  in idma64_prep_slave_sg()
335 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_issue_pending()  local
338 	spin_lock_irqsave(&idma64c->vchan.lock, flags);  in idma64_issue_pending()
339 	if (vchan_issue_pending(&idma64c->vchan) && !idma64c->desc)  in idma64_issue_pending()
340 		idma64_start_transfer(idma64c);  in idma64_issue_pending()
341 	spin_unlock_irqrestore(&idma64c->vchan.lock, flags);  in idma64_issue_pending()
344 static size_t idma64_active_desc_size(struct idma64_chan *idma64c)  in idma64_active_desc_size()  argument
346 	struct idma64_desc *desc = idma64c->desc;  in idma64_active_desc_size()
349 	u64 llp = channel_readq(idma64c, LLP);  in idma64_active_desc_size()
350 	u32 ctlhi = channel_readl(idma64c, CTL_HI);  in idma64_active_desc_size()
372 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_tx_status()  local
382 	spin_lock_irqsave(&idma64c->vchan.lock, flags);  in idma64_tx_status()
383 	vdesc = vchan_find_desc(&idma64c->vchan, cookie);  in idma64_tx_status()
384 	if (idma64c->desc && cookie == idma64c->desc->vdesc.tx.cookie) {  in idma64_tx_status()
385 		bytes = idma64_active_desc_size(idma64c);  in idma64_tx_status()
387 		status = idma64c->desc->status;  in idma64_tx_status()
392 	spin_unlock_irqrestore(&idma64c->vchan.lock, flags);  in idma64_tx_status()
408 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_slave_config()  local
410 	memcpy(&idma64c->config, config, sizeof(idma64c->config));  in idma64_slave_config()
412 	convert_burst(&idma64c->config.src_maxburst);  in idma64_slave_config()
413 	convert_burst(&idma64c->config.dst_maxburst);  in idma64_slave_config()
418 static void idma64_chan_deactivate(struct idma64_chan *idma64c, bool drain)  in idma64_chan_deactivate()  argument
423 	cfglo = channel_readl(idma64c, CFG_LO);  in idma64_chan_deactivate()
429 	channel_writel(idma64c, CFG_LO, cfglo | IDMA64C_CFGL_CH_SUSP);  in idma64_chan_deactivate()
432 		cfglo = channel_readl(idma64c, CFG_LO);  in idma64_chan_deactivate()
436 static void idma64_chan_activate(struct idma64_chan *idma64c)  in idma64_chan_activate()  argument
440 	cfglo = channel_readl(idma64c, CFG_LO);  in idma64_chan_activate()
441 	channel_writel(idma64c, CFG_LO, cfglo & ~IDMA64C_CFGL_CH_SUSP);  in idma64_chan_activate()
446 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_pause()  local
449 	spin_lock_irqsave(&idma64c->vchan.lock, flags);  in idma64_pause()
450 	if (idma64c->desc && idma64c->desc->status == DMA_IN_PROGRESS) {  in idma64_pause()
451 		idma64_chan_deactivate(idma64c, false);  in idma64_pause()
452 		idma64c->desc->status = DMA_PAUSED;  in idma64_pause()
454 	spin_unlock_irqrestore(&idma64c->vchan.lock, flags);  in idma64_pause()
461 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_resume()  local
464 	spin_lock_irqsave(&idma64c->vchan.lock, flags);  in idma64_resume()
465 	if (idma64c->desc && idma64c->desc->status == DMA_PAUSED) {  in idma64_resume()
466 		idma64c->desc->status = DMA_IN_PROGRESS;  in idma64_resume()
467 		idma64_chan_activate(idma64c);  in idma64_resume()
469 	spin_unlock_irqrestore(&idma64c->vchan.lock, flags);  in idma64_resume()
476 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_terminate_all()  local
480 	spin_lock_irqsave(&idma64c->vchan.lock, flags);  in idma64_terminate_all()
481 	idma64_chan_deactivate(idma64c, true);  in idma64_terminate_all()
482 	idma64_stop_transfer(idma64c);  in idma64_terminate_all()
483 	if (idma64c->desc) {  in idma64_terminate_all()
484 		idma64_vdesc_free(&idma64c->desc->vdesc);  in idma64_terminate_all()
485 		idma64c->desc = NULL;  in idma64_terminate_all()
487 	vchan_get_all_descriptors(&idma64c->vchan, &head);  in idma64_terminate_all()
488 	spin_unlock_irqrestore(&idma64c->vchan.lock, flags);  in idma64_terminate_all()
490 	vchan_dma_desc_free_list(&idma64c->vchan, &head);  in idma64_terminate_all()
496 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_synchronize()  local
498 	vchan_synchronize(&idma64c->vchan);  in idma64_synchronize()
503 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_alloc_chan_resources()  local
506 	idma64c->pool = dma_pool_create(dev_name(chan2dev(chan)),  in idma64_alloc_chan_resources()
509 	if (!idma64c->pool) {  in idma64_alloc_chan_resources()
519 	struct idma64_chan *idma64c = to_idma64_chan(chan);  in idma64_free_chan_resources()  local
522 	dma_pool_destroy(idma64c->pool);  in idma64_free_chan_resources()
523 	idma64c->pool = NULL;  in idma64_free_chan_resources()
564 		struct idma64_chan *idma64c = &idma64->chan[i];  in idma64_probe()  local
566 		idma64c->vchan.desc_free = idma64_vdesc_free;  in idma64_probe()
567 		vchan_init(&idma64c->vchan, &idma64->dma);  in idma64_probe()
569 		idma64c->regs = idma64->regs + i * IDMA64_CH_LENGTH;  in idma64_probe()
570 		idma64c->mask = BIT(i);  in idma64_probe()
621 		struct idma64_chan *idma64c = &idma64->chan[i];  in idma64_remove()  local
623 		tasklet_kill(&idma64c->vchan.task);  in idma64_remove()