1 // SPDX-License-Identifier: GPL-2.0+
3 // Copyright (c) 2013-2014 Freescale Semiconductor, Inc
9 #include <linux/dma-mapping.h>
11 #include "fsl-edma-common.h"
47 	struct edma_regs *regs = &fsl_chan->edma->regs;  in fsl_edma_enable_request()
48 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_enable_request()
50 	if (fsl_chan->edma->drvdata->version == v1) {  in fsl_edma_enable_request()
51 		edma_writeb(fsl_chan->edma, EDMA_SEEI_SEEI(ch), regs->seei);  in fsl_edma_enable_request()
52 		edma_writeb(fsl_chan->edma, ch, regs->serq);  in fsl_edma_enable_request()
54 		/* ColdFire is big endian, and accesses natively  in fsl_edma_enable_request()
55 		 * big endian I/O peripherals  in fsl_edma_enable_request()
57 		iowrite8(EDMA_SEEI_SEEI(ch), regs->seei);  in fsl_edma_enable_request()
58 		iowrite8(ch, regs->serq);  in fsl_edma_enable_request()
64 	struct edma_regs *regs = &fsl_chan->edma->regs;  in fsl_edma_disable_request()
65 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_disable_request()
67 	if (fsl_chan->edma->drvdata->version == v1) {  in fsl_edma_disable_request()
68 		edma_writeb(fsl_chan->edma, ch, regs->cerq);  in fsl_edma_disable_request()
69 		edma_writeb(fsl_chan->edma, EDMA_CEEI_CEEI(ch), regs->ceei);  in fsl_edma_disable_request()
71 		/* ColdFire is big endian, and accesses natively  in fsl_edma_disable_request()
72 		 * big endian I/O peripherals  in fsl_edma_disable_request()
74 		iowrite8(ch, regs->cerq);  in fsl_edma_disable_request()
75 		iowrite8(EDMA_CEEI_CEEI(ch), regs->ceei);  in fsl_edma_disable_request()
109 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_chan_mux()
112 	int endian_diff[4] = {3, 1, -1, -3};  in fsl_edma_chan_mux()
113 	u32 dmamux_nr = fsl_chan->edma->drvdata->dmamuxs;  in fsl_edma_chan_mux()
115 	chans_per_mux = fsl_chan->edma->n_chans / dmamux_nr;  in fsl_edma_chan_mux()
116 	ch_off = fsl_chan->vchan.chan.chan_id % chans_per_mux;  in fsl_edma_chan_mux()
118 	if (fsl_chan->edma->drvdata->mux_swap)  in fsl_edma_chan_mux()
121 	muxaddr = fsl_chan->edma->muxbase[ch / chans_per_mux];  in fsl_edma_chan_mux()
124 	if (fsl_chan->edma->drvdata->version == v3)  in fsl_edma_chan_mux()
153 	for (i = 0; i < fsl_desc->n_tcds; i++)  in fsl_edma_free_desc()
154 		dma_pool_free(fsl_desc->echan->tcd_pool, fsl_desc->tcd[i].vtcd,  in fsl_edma_free_desc()
155 			      fsl_desc->tcd[i].ptcd);  in fsl_edma_free_desc()
166 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_terminate_all()
168 	fsl_chan->edesc = NULL;  in fsl_edma_terminate_all()
169 	fsl_chan->idle = true;  in fsl_edma_terminate_all()
170 	vchan_get_all_descriptors(&fsl_chan->vchan, &head);  in fsl_edma_terminate_all()
171 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_terminate_all()
172 	vchan_dma_desc_free_list(&fsl_chan->vchan, &head);  in fsl_edma_terminate_all()
182 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_pause()
183 	if (fsl_chan->edesc) {  in fsl_edma_pause()
185 		fsl_chan->status = DMA_PAUSED;  in fsl_edma_pause()
186 		fsl_chan->idle = true;  in fsl_edma_pause()
188 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_pause()
198 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_resume()
199 	if (fsl_chan->edesc) {  in fsl_edma_resume()
201 		fsl_chan->status = DMA_IN_PROGRESS;  in fsl_edma_resume()
202 		fsl_chan->idle = false;  in fsl_edma_resume()
204 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_resume()
211 	if (fsl_chan->dma_dir != DMA_NONE)  in fsl_edma_unprep_slave_dma()
212 		dma_unmap_resource(fsl_chan->vchan.chan.device->dev,  in fsl_edma_unprep_slave_dma()
213 				   fsl_chan->dma_dev_addr,  in fsl_edma_unprep_slave_dma()
214 				   fsl_chan->dma_dev_size,  in fsl_edma_unprep_slave_dma()
215 				   fsl_chan->dma_dir, 0);  in fsl_edma_unprep_slave_dma()
216 	fsl_chan->dma_dir = DMA_NONE;  in fsl_edma_unprep_slave_dma()
222 	struct device *dev = fsl_chan->vchan.chan.device->dev;  in fsl_edma_prep_slave_dma()
230 		addr = fsl_chan->cfg.dst_addr;  in fsl_edma_prep_slave_dma()
231 		size = fsl_chan->cfg.dst_maxburst;  in fsl_edma_prep_slave_dma()
235 		addr = fsl_chan->cfg.src_addr;  in fsl_edma_prep_slave_dma()
236 		size = fsl_chan->cfg.src_maxburst;  in fsl_edma_prep_slave_dma()
244 	if (fsl_chan->dma_dir == dma_dir)  in fsl_edma_prep_slave_dma()
249 	fsl_chan->dma_dev_addr = dma_map_resource(dev, addr, size, dma_dir, 0);  in fsl_edma_prep_slave_dma()
250 	if (dma_mapping_error(dev, fsl_chan->dma_dev_addr))  in fsl_edma_prep_slave_dma()
252 	fsl_chan->dma_dev_size = size;  in fsl_edma_prep_slave_dma()
253 	fsl_chan->dma_dir = dma_dir;  in fsl_edma_prep_slave_dma()
263 	memcpy(&fsl_chan->cfg, cfg, sizeof(*cfg));  in fsl_edma_slave_config()
273 	struct fsl_edma_desc *edesc = fsl_chan->edesc;  in fsl_edma_desc_residue()
274 	struct edma_regs *regs = &fsl_chan->edma->regs;  in fsl_edma_desc_residue()
275 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_desc_residue()
276 	enum dma_transfer_direction dir = edesc->dirn;  in fsl_edma_desc_residue()
281 	/* calculate the total size in this desc */  in fsl_edma_desc_residue()
282 	for (len = i = 0; i < fsl_chan->edesc->n_tcds; i++)  in fsl_edma_desc_residue()
283 		len += le32_to_cpu(edesc->tcd[i].vtcd->nbytes)  in fsl_edma_desc_residue()
284 			* le16_to_cpu(edesc->tcd[i].vtcd->biter);  in fsl_edma_desc_residue()
290 		cur_addr = edma_readl(fsl_chan->edma, &regs->tcd[ch].saddr);  in fsl_edma_desc_residue()
292 		cur_addr = edma_readl(fsl_chan->edma, &regs->tcd[ch].daddr);  in fsl_edma_desc_residue()
295 	for (i = 0; i < fsl_chan->edesc->n_tcds; i++) {  in fsl_edma_desc_residue()
296 		size = le32_to_cpu(edesc->tcd[i].vtcd->nbytes)  in fsl_edma_desc_residue()
297 			* le16_to_cpu(edesc->tcd[i].vtcd->biter);  in fsl_edma_desc_residue()
299 			dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->saddr);  in fsl_edma_desc_residue()
301 			dma_addr = le32_to_cpu(edesc->tcd[i].vtcd->daddr);  in fsl_edma_desc_residue()
303 		len -= size;  in fsl_edma_desc_residue()
305 			len += dma_addr + size - cur_addr;  in fsl_edma_desc_residue()
326 		return fsl_chan->status;  in fsl_edma_tx_status()
328 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_tx_status()
329 	vdesc = vchan_find_desc(&fsl_chan->vchan, cookie);  in fsl_edma_tx_status()
330 	if (fsl_chan->edesc && cookie == fsl_chan->edesc->vdesc.tx.cookie)  in fsl_edma_tx_status()
331 		txstate->residue =  in fsl_edma_tx_status()
334 		txstate->residue =  in fsl_edma_tx_status()
337 		txstate->residue = 0;  in fsl_edma_tx_status()
339 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_tx_status()
341 	return fsl_chan->status;  in fsl_edma_tx_status()
348 	struct fsl_edma_engine *edma = fsl_chan->edma;  in fsl_edma_set_tcd_regs()
349 	struct edma_regs *regs = &fsl_chan->edma->regs;  in fsl_edma_set_tcd_regs()
350 	u32 ch = fsl_chan->vchan.chan.chan_id;  in fsl_edma_set_tcd_regs()
355 	 * endian format. However, we need to load the TCD registers in  in fsl_edma_set_tcd_regs()
356 	 * big- or little-endian obeying the eDMA engine model endian,  in fsl_edma_set_tcd_regs()
359 	edma_writew(edma, 0,  &regs->tcd[ch].csr);  in fsl_edma_set_tcd_regs()
361 	edma_writel(edma, (s32)tcd->saddr, &regs->tcd[ch].saddr);  in fsl_edma_set_tcd_regs()
362 	edma_writel(edma, (s32)tcd->daddr, &regs->tcd[ch].daddr);  in fsl_edma_set_tcd_regs()
364 	edma_writew(edma, (s16)tcd->attr, &regs->tcd[ch].attr);  in fsl_edma_set_tcd_regs()
365 	edma_writew(edma, tcd->soff, &regs->tcd[ch].soff);  in fsl_edma_set_tcd_regs()
367 	edma_writel(edma, (s32)tcd->nbytes, &regs->tcd[ch].nbytes);  in fsl_edma_set_tcd_regs()
368 	edma_writel(edma, (s32)tcd->slast, &regs->tcd[ch].slast);  in fsl_edma_set_tcd_regs()
370 	edma_writew(edma, (s16)tcd->citer, &regs->tcd[ch].citer);  in fsl_edma_set_tcd_regs()
371 	edma_writew(edma, (s16)tcd->biter, &regs->tcd[ch].biter);  in fsl_edma_set_tcd_regs()
372 	edma_writew(edma, (s16)tcd->doff, &regs->tcd[ch].doff);  in fsl_edma_set_tcd_regs()
374 	edma_writel(edma, (s32)tcd->dlast_sga,  in fsl_edma_set_tcd_regs()
375 			&regs->tcd[ch].dlast_sga);  in fsl_edma_set_tcd_regs()
377 	if (fsl_chan->is_sw) {  in fsl_edma_set_tcd_regs()
378 		csr = le16_to_cpu(tcd->csr);  in fsl_edma_set_tcd_regs()
380 		tcd->csr = cpu_to_le16(csr);  in fsl_edma_set_tcd_regs()
383 	edma_writew(edma, (s16)tcd->csr, &regs->tcd[ch].csr);  in fsl_edma_set_tcd_regs()
396 	 * endian format irrespective of the register endian model.  in fsl_edma_fill_tcd()
397 	 * So we put the value in little endian in memory, waiting  in fsl_edma_fill_tcd()
400 	tcd->saddr = cpu_to_le32(src);  in fsl_edma_fill_tcd()
401 	tcd->daddr = cpu_to_le32(dst);  in fsl_edma_fill_tcd()
403 	tcd->attr = cpu_to_le16(attr);  in fsl_edma_fill_tcd()
405 	tcd->soff = cpu_to_le16(soff);  in fsl_edma_fill_tcd()
407 	tcd->nbytes = cpu_to_le32(nbytes);  in fsl_edma_fill_tcd()
408 	tcd->slast = cpu_to_le32(slast);  in fsl_edma_fill_tcd()
410 	tcd->citer = cpu_to_le16(EDMA_TCD_CITER_CITER(citer));  in fsl_edma_fill_tcd()
411 	tcd->doff = cpu_to_le16(doff);  in fsl_edma_fill_tcd()
413 	tcd->dlast_sga = cpu_to_le32(dlast_sga);  in fsl_edma_fill_tcd()
415 	tcd->biter = cpu_to_le16(EDMA_TCD_BITER_BITER(biter));  in fsl_edma_fill_tcd()
425 	tcd->csr = cpu_to_le16(csr);  in fsl_edma_fill_tcd()
438 	fsl_desc->echan = fsl_chan;  in fsl_edma_alloc_desc()
439 	fsl_desc->n_tcds = sg_len;  in fsl_edma_alloc_desc()
441 		fsl_desc->tcd[i].vtcd = dma_pool_alloc(fsl_chan->tcd_pool,  in fsl_edma_alloc_desc()
442 					GFP_NOWAIT, &fsl_desc->tcd[i].ptcd);  in fsl_edma_alloc_desc()
443 		if (!fsl_desc->tcd[i].vtcd)  in fsl_edma_alloc_desc()
449 	while (--i >= 0)  in fsl_edma_alloc_desc()
450 		dma_pool_free(fsl_chan->tcd_pool, fsl_desc->tcd[i].vtcd,  in fsl_edma_alloc_desc()
451 				fsl_desc->tcd[i].ptcd);  in fsl_edma_alloc_desc()
478 	fsl_desc->iscyclic = true;  in fsl_edma_prep_dma_cyclic()
479 	fsl_desc->dirn = direction;  in fsl_edma_prep_dma_cyclic()
483 		fsl_chan->attr =  in fsl_edma_prep_dma_cyclic()
484 			fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width);  in fsl_edma_prep_dma_cyclic()
485 		nbytes = fsl_chan->cfg.dst_addr_width *  in fsl_edma_prep_dma_cyclic()
486 			fsl_chan->cfg.dst_maxburst;  in fsl_edma_prep_dma_cyclic()
488 		fsl_chan->attr =  in fsl_edma_prep_dma_cyclic()
489 			fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width);  in fsl_edma_prep_dma_cyclic()
490 		nbytes = fsl_chan->cfg.src_addr_width *  in fsl_edma_prep_dma_cyclic()
491 			fsl_chan->cfg.src_maxburst;  in fsl_edma_prep_dma_cyclic()
501 		last_sg = fsl_desc->tcd[(i + 1) % sg_len].ptcd;  in fsl_edma_prep_dma_cyclic()
505 			dst_addr = fsl_chan->dma_dev_addr;  in fsl_edma_prep_dma_cyclic()
506 			soff = fsl_chan->cfg.dst_addr_width;  in fsl_edma_prep_dma_cyclic()
509 			src_addr = fsl_chan->dma_dev_addr;  in fsl_edma_prep_dma_cyclic()
512 			doff = fsl_chan->cfg.src_addr_width;  in fsl_edma_prep_dma_cyclic()
515 		fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr, dst_addr,  in fsl_edma_prep_dma_cyclic()
516 				  fsl_chan->attr, soff, nbytes, 0, iter,  in fsl_edma_prep_dma_cyclic()
521 	return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);  in fsl_edma_prep_dma_cyclic()
546 	fsl_desc->iscyclic = false;  in fsl_edma_prep_slave_sg()
547 	fsl_desc->dirn = direction;  in fsl_edma_prep_slave_sg()
550 		fsl_chan->attr =  in fsl_edma_prep_slave_sg()
551 			fsl_edma_get_tcd_attr(fsl_chan->cfg.dst_addr_width);  in fsl_edma_prep_slave_sg()
552 		nbytes = fsl_chan->cfg.dst_addr_width *  in fsl_edma_prep_slave_sg()
553 			fsl_chan->cfg.dst_maxburst;  in fsl_edma_prep_slave_sg()
555 		fsl_chan->attr =  in fsl_edma_prep_slave_sg()
556 			fsl_edma_get_tcd_attr(fsl_chan->cfg.src_addr_width);  in fsl_edma_prep_slave_sg()
557 		nbytes = fsl_chan->cfg.src_addr_width *  in fsl_edma_prep_slave_sg()
558 			fsl_chan->cfg.src_maxburst;  in fsl_edma_prep_slave_sg()
564 			dst_addr = fsl_chan->dma_dev_addr;  in fsl_edma_prep_slave_sg()
565 			soff = fsl_chan->cfg.dst_addr_width;  in fsl_edma_prep_slave_sg()
568 			src_addr = fsl_chan->dma_dev_addr;  in fsl_edma_prep_slave_sg()
571 			doff = fsl_chan->cfg.src_addr_width;  in fsl_edma_prep_slave_sg()
575 		if (i < sg_len - 1) {  in fsl_edma_prep_slave_sg()
576 			last_sg = fsl_desc->tcd[(i + 1)].ptcd;  in fsl_edma_prep_slave_sg()
577 			fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr,  in fsl_edma_prep_slave_sg()
578 					  dst_addr, fsl_chan->attr, soff,  in fsl_edma_prep_slave_sg()
583 			fsl_edma_fill_tcd(fsl_desc->tcd[i].vtcd, src_addr,  in fsl_edma_prep_slave_sg()
584 					  dst_addr, fsl_chan->attr, soff,  in fsl_edma_prep_slave_sg()
590 	return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);  in fsl_edma_prep_slave_sg()
604 	fsl_desc->iscyclic = false;  in fsl_edma_prep_memcpy()
606 	fsl_chan->is_sw = true;  in fsl_edma_prep_memcpy()
609 	fsl_edma_fill_tcd(fsl_desc->tcd[0].vtcd, dma_src, dma_dst,  in fsl_edma_prep_memcpy()
613 	return vchan_tx_prep(&fsl_chan->vchan, &fsl_desc->vdesc, flags);  in fsl_edma_prep_memcpy()
621 	lockdep_assert_held(&fsl_chan->vchan.lock);  in fsl_edma_xfer_desc()
623 	vdesc = vchan_next_desc(&fsl_chan->vchan);  in fsl_edma_xfer_desc()
626 	fsl_chan->edesc = to_fsl_edma_desc(vdesc);  in fsl_edma_xfer_desc()
627 	fsl_edma_set_tcd_regs(fsl_chan, fsl_chan->edesc->tcd[0].vtcd);  in fsl_edma_xfer_desc()
629 	fsl_chan->status = DMA_IN_PROGRESS;  in fsl_edma_xfer_desc()
630 	fsl_chan->idle = false;  in fsl_edma_xfer_desc()
639 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_issue_pending()
641 	if (unlikely(fsl_chan->pm_state != RUNNING)) {  in fsl_edma_issue_pending()
642 		spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_issue_pending()
647 	if (vchan_issue_pending(&fsl_chan->vchan) && !fsl_chan->edesc)  in fsl_edma_issue_pending()
650 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_issue_pending()
658 	fsl_chan->tcd_pool = dma_pool_create("tcd_pool", chan->device->dev,  in fsl_edma_alloc_chan_resources()
668 	struct fsl_edma_engine *edma = fsl_chan->edma;  in fsl_edma_free_chan_resources()
672 	spin_lock_irqsave(&fsl_chan->vchan.lock, flags);  in fsl_edma_free_chan_resources()
674 	if (edma->drvdata->dmamuxs)  in fsl_edma_free_chan_resources()
676 	fsl_chan->edesc = NULL;  in fsl_edma_free_chan_resources()
677 	vchan_get_all_descriptors(&fsl_chan->vchan, &head);  in fsl_edma_free_chan_resources()
679 	spin_unlock_irqrestore(&fsl_chan->vchan.lock, flags);  in fsl_edma_free_chan_resources()
681 	vchan_dma_desc_free_list(&fsl_chan->vchan, &head);  in fsl_edma_free_chan_resources()
682 	dma_pool_destroy(fsl_chan->tcd_pool);  in fsl_edma_free_chan_resources()
683 	fsl_chan->tcd_pool = NULL;  in fsl_edma_free_chan_resources()
684 	fsl_chan->is_sw = false;  in fsl_edma_free_chan_resources()
693 				&dmadev->channels, vchan.chan.device_node) {  in fsl_edma_cleanup_vchan()
694 		list_del(&chan->vchan.chan.device_node);  in fsl_edma_cleanup_vchan()
695 		tasklet_kill(&chan->vchan.task);  in fsl_edma_cleanup_vchan()
711 	edma->regs.cr = edma->membase + EDMA_CR;  in fsl_edma_setup_regs()
712 	edma->regs.es = edma->membase + EDMA_ES;  in fsl_edma_setup_regs()
713 	edma->regs.erql = edma->membase + EDMA_ERQ;  in fsl_edma_setup_regs()
714 	edma->regs.eeil = edma->membase + EDMA_EEI;  in fsl_edma_setup_regs()
716 	edma->regs.serq = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
718 	edma->regs.cerq = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
720 	edma->regs.seei = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
722 	edma->regs.ceei = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
724 	edma->regs.cint = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
726 	edma->regs.cerr = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
728 	edma->regs.ssrt = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
730 	edma->regs.cdne = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
732 	edma->regs.intl = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
734 	edma->regs.errl = edma->membase + ((edma->drvdata->version == v2) ?  in fsl_edma_setup_regs()
737 	if (edma->drvdata->version == v2) {  in fsl_edma_setup_regs()
738 		edma->regs.erqh = edma->membase + EDMA64_ERQH;  in fsl_edma_setup_regs()
739 		edma->regs.eeih = edma->membase + EDMA64_EEIH;  in fsl_edma_setup_regs()
740 		edma->regs.errh = edma->membase + EDMA64_ERRH;  in fsl_edma_setup_regs()
741 		edma->regs.inth = edma->membase + EDMA64_INTH;  in fsl_edma_setup_regs()
744 	edma->regs.tcd = edma->membase + EDMA_TCD;  in fsl_edma_setup_regs()