31 static int st_fdma_dreq_get(struct st_fdma_chan *fchan)  in st_fdma_dreq_get()  argument
33 	struct st_fdma_dev *fdev = fchan->fdev;  in st_fdma_dreq_get()
34 	u32 req_line_cfg = fchan->cfg.req_line;  in st_fdma_dreq_get()
65 static void st_fdma_dreq_put(struct st_fdma_chan *fchan)  in st_fdma_dreq_put()  argument
67 	struct st_fdma_dev *fdev = fchan->fdev;  in st_fdma_dreq_put()
69 	dev_dbg(fdev->dev, "put dreq_line:%#x\n", fchan->dreq_line);  in st_fdma_dreq_put()
70 	clear_bit(fchan->dreq_line, &fdev->dreq_mask);  in st_fdma_dreq_put()
73 static void st_fdma_xfer_desc(struct st_fdma_chan *fchan)  in st_fdma_xfer_desc()  argument
78 	vdesc = vchan_next_desc(&fchan->vchan);  in st_fdma_xfer_desc()
82 	fchan->fdesc = to_st_fdma_desc(vdesc);  in st_fdma_xfer_desc()
83 	nbytes = fchan->fdesc->node[0].desc->nbytes;  in st_fdma_xfer_desc()
84 	cmd = FDMA_CMD_START(fchan->vchan.chan.chan_id);  in st_fdma_xfer_desc()
85 	ch_cmd = fchan->fdesc->node[0].pdesc | FDMA_CH_CMD_STA_START;  in st_fdma_xfer_desc()
88 	fnode_write(fchan, nbytes, FDMA_CNTN_OFST);  in st_fdma_xfer_desc()
89 	fchan_write(fchan, ch_cmd, FDMA_CH_CMD_OFST);  in st_fdma_xfer_desc()
91 		fchan->fdev->slim_rproc->peri + FDMA_CMD_SET_OFST);  in st_fdma_xfer_desc()
93 	dev_dbg(fchan->fdev->dev, "start chan:%d\n", fchan->vchan.chan.chan_id);  in st_fdma_xfer_desc()
96 static void st_fdma_ch_sta_update(struct st_fdma_chan *fchan,  in st_fdma_ch_sta_update()  argument
100 	int ch_id = fchan->vchan.chan.chan_id;  in st_fdma_ch_sta_update()
101 	struct st_fdma_dev *fdev = fchan->fdev;  in st_fdma_ch_sta_update()
103 	ch_sta = fchan_read(fchan, FDMA_CH_CMD_OFST);  in st_fdma_ch_sta_update()
109 		fchan->status = DMA_ERROR;  in st_fdma_ch_sta_update()
115 		fchan->status = DMA_PAUSED;  in st_fdma_ch_sta_update()
119 		fchan->status = DMA_IN_PROGRESS;  in st_fdma_ch_sta_update()
128 	struct st_fdma_chan *fchan = &fdev->chans[0];  in st_fdma_irq_handler()  local
134 	for (; int_sta != 0 ; int_sta >>= 2, fchan++) {  in st_fdma_irq_handler()
138 		spin_lock(&fchan->vchan.lock);  in st_fdma_irq_handler()
139 		st_fdma_ch_sta_update(fchan, int_sta);  in st_fdma_irq_handler()
141 		if (fchan->fdesc) {  in st_fdma_irq_handler()
142 			if (!fchan->fdesc->iscyclic) {  in st_fdma_irq_handler()
143 				list_del(&fchan->fdesc->vdesc.node);  in st_fdma_irq_handler()
144 				vchan_cookie_complete(&fchan->fdesc->vdesc);  in st_fdma_irq_handler()
145 				fchan->fdesc = NULL;  in st_fdma_irq_handler()
146 				fchan->status = DMA_COMPLETE;  in st_fdma_irq_handler()
148 				vchan_cyclic_callback(&fchan->fdesc->vdesc);  in st_fdma_irq_handler()
152 			if (!fchan->fdesc)  in st_fdma_irq_handler()
153 				st_fdma_xfer_desc(fchan);  in st_fdma_irq_handler()
156 		spin_unlock(&fchan->vchan.lock);  in st_fdma_irq_handler()
170 	struct st_fdma_chan *fchan;  in st_fdma_of_xlate()  local
189 	fchan = to_st_fdma_chan(chan);  in st_fdma_of_xlate()
191 	fchan->cfg.of_node = dma_spec->np;  in st_fdma_of_xlate()
192 	fchan->cfg.req_line = dma_spec->args[0];  in st_fdma_of_xlate()
193 	fchan->cfg.req_ctrl = 0;  in st_fdma_of_xlate()
194 	fchan->cfg.type = ST_FDMA_TYPE_FREE_RUN;  in st_fdma_of_xlate()
197 		fchan->cfg.req_ctrl = dma_spec->args[1]  in st_fdma_of_xlate()
201 		fchan->cfg.type = dma_spec->args[2];  in st_fdma_of_xlate()
203 	if (fchan->cfg.type == ST_FDMA_TYPE_FREE_RUN) {  in st_fdma_of_xlate()
204 		fchan->dreq_line = 0;  in st_fdma_of_xlate()
206 		fchan->dreq_line = st_fdma_dreq_get(fchan);  in st_fdma_of_xlate()
207 		if (IS_ERR_VALUE(fchan->dreq_line)) {  in st_fdma_of_xlate()
208 			chan = ERR_PTR(fchan->dreq_line);  in st_fdma_of_xlate()
214 		fchan->cfg.req_line, fchan->cfg.type, fchan->cfg.req_ctrl);  in st_fdma_of_xlate()
231 		dma_pool_free(fdesc->fchan->node_pool, fdesc->node[i].desc,  in st_fdma_free_desc()
236 static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan,  in st_fdma_alloc_desc()  argument
246 	fdesc->fchan = fchan;  in st_fdma_alloc_desc()
249 		fdesc->node[i].desc = dma_pool_alloc(fchan->node_pool,  in st_fdma_alloc_desc()
258 		dma_pool_free(fchan->node_pool, fdesc->node[i].desc,  in st_fdma_alloc_desc()
266 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_alloc_chan_res()  local
269 	fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device),  in st_fdma_alloc_chan_res()
270 					    fchan->fdev->dev,  in st_fdma_alloc_chan_res()
275 	if (!fchan->node_pool) {  in st_fdma_alloc_chan_res()
276 		dev_err(fchan->fdev->dev, "unable to allocate desc pool\n");  in st_fdma_alloc_chan_res()
280 	dev_dbg(fchan->fdev->dev, "alloc ch_id:%d type:%d\n",  in st_fdma_alloc_chan_res()
281 		fchan->vchan.chan.chan_id, fchan->cfg.type);  in st_fdma_alloc_chan_res()
288 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_free_chan_res()  local
289 	struct rproc *rproc = fchan->fdev->slim_rproc->rproc;  in st_fdma_free_chan_res()
292 	dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n",  in st_fdma_free_chan_res()
293 		__func__, fchan->vchan.chan.chan_id);  in st_fdma_free_chan_res()
295 	if (fchan->cfg.type != ST_FDMA_TYPE_FREE_RUN)  in st_fdma_free_chan_res()
296 		st_fdma_dreq_put(fchan);  in st_fdma_free_chan_res()
298 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_free_chan_res()
299 	fchan->fdesc = NULL;  in st_fdma_free_chan_res()
300 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_free_chan_res()
302 	dma_pool_destroy(fchan->node_pool);  in st_fdma_free_chan_res()
303 	fchan->node_pool = NULL;  in st_fdma_free_chan_res()
304 	memset(&fchan->cfg, 0, sizeof(struct st_fdma_cfg));  in st_fdma_free_chan_res()
313 	struct st_fdma_chan *fchan;  in st_fdma_prep_dma_memcpy()  local
320 	fchan = to_st_fdma_chan(chan);  in st_fdma_prep_dma_memcpy()
323 	fdesc = st_fdma_alloc_desc(fchan, 1);  in st_fdma_prep_dma_memcpy()
325 		dev_err(fchan->fdev->dev, "no memory for desc\n");  in st_fdma_prep_dma_memcpy()
342 	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);  in st_fdma_prep_dma_memcpy()
345 static int config_reqctrl(struct st_fdma_chan *fchan,  in config_reqctrl()  argument
350 	int ch_id = fchan->vchan.chan.chan_id;  in config_reqctrl()
351 	struct st_fdma_dev *fdev = fchan->fdev;  in config_reqctrl()
356 		fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_WNR;  in config_reqctrl()
357 		maxburst = fchan->scfg.src_maxburst;  in config_reqctrl()
358 		width = fchan->scfg.src_addr_width;  in config_reqctrl()
359 		addr = fchan->scfg.src_addr;  in config_reqctrl()
363 		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_WNR;  in config_reqctrl()
364 		maxburst = fchan->scfg.dst_maxburst;  in config_reqctrl()
365 		width = fchan->scfg.dst_addr_width;  in config_reqctrl()
366 		addr = fchan->scfg.dst_addr;  in config_reqctrl()
373 	fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_OPCODE_MASK;  in config_reqctrl()
378 		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST1;  in config_reqctrl()
382 		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST2;  in config_reqctrl()
386 		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST4;  in config_reqctrl()
390 		fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST8;  in config_reqctrl()
397 	fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_NUM_OPS_MASK;  in config_reqctrl()
398 	fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_NUM_OPS(maxburst-1);  in config_reqctrl()
399 	dreq_write(fchan, fchan->cfg.req_ctrl, FDMA_REQ_CTRL_OFST);  in config_reqctrl()
401 	fchan->cfg.dev_addr = addr;  in config_reqctrl()
402 	fchan->cfg.dir = direction;  in config_reqctrl()
405 		ch_id, addr, fchan->cfg.req_ctrl);  in config_reqctrl()
411 			struct st_fdma_chan *fchan,  in fill_hw_node()  argument
417 		hw_node->daddr = fchan->cfg.dev_addr;  in fill_hw_node()
421 		hw_node->saddr = fchan->cfg.dev_addr;  in fill_hw_node()
431 	struct st_fdma_chan *fchan;  in st_fdma_prep_common()  local
436 	fchan = to_st_fdma_chan(chan);  in st_fdma_prep_common()
439 		dev_err(fchan->fdev->dev, "bad direction?\n");  in st_fdma_prep_common()
443 	return fchan;  in st_fdma_prep_common()
451 	struct st_fdma_chan *fchan;  in st_fdma_prep_dma_cyclic()  local
455 	fchan = st_fdma_prep_common(chan, len, direction);  in st_fdma_prep_dma_cyclic()
456 	if (!fchan)  in st_fdma_prep_dma_cyclic()
462 	if (config_reqctrl(fchan, direction)) {  in st_fdma_prep_dma_cyclic()
463 		dev_err(fchan->fdev->dev, "bad width or direction\n");  in st_fdma_prep_dma_cyclic()
469 		dev_err(fchan->fdev->dev, "len is not multiple of period\n");  in st_fdma_prep_dma_cyclic()
474 	fdesc = st_fdma_alloc_desc(fchan, sg_len);  in st_fdma_prep_dma_cyclic()
476 		dev_err(fchan->fdev->dev, "no memory for desc\n");  in st_fdma_prep_dma_cyclic()
488 			FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);  in st_fdma_prep_dma_cyclic()
491 		fill_hw_node(hw_node, fchan, direction);  in st_fdma_prep_dma_cyclic()
502 	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);  in st_fdma_prep_dma_cyclic()
510 	struct st_fdma_chan *fchan;  in st_fdma_prep_slave_sg()  local
516 	fchan = st_fdma_prep_common(chan, sg_len, direction);  in st_fdma_prep_slave_sg()
517 	if (!fchan)  in st_fdma_prep_slave_sg()
523 	fdesc = st_fdma_alloc_desc(fchan, sg_len);  in st_fdma_prep_slave_sg()
525 		dev_err(fchan->fdev->dev, "no memory for desc\n");  in st_fdma_prep_slave_sg()
535 		hw_node->control = FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);  in st_fdma_prep_slave_sg()
537 		fill_hw_node(hw_node, fchan, direction);  in st_fdma_prep_slave_sg()
551 	return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);  in st_fdma_prep_slave_sg()
554 static size_t st_fdma_desc_residue(struct st_fdma_chan *fchan,  in st_fdma_desc_residue()  argument
558 	struct st_fdma_desc *fdesc = fchan->fdesc;  in st_fdma_desc_residue()
564 		cur_addr = fchan_read(fchan, FDMA_CH_CMD_OFST);  in st_fdma_desc_residue()
568 	for (i = fchan->fdesc->n_nodes - 1 ; i >= 0; i--) {  in st_fdma_desc_residue()
570 			residue += fnode_read(fchan, FDMA_CNTN_OFST);  in st_fdma_desc_residue()
583 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_tx_status()  local
592 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_tx_status()
593 	vd = vchan_find_desc(&fchan->vchan, cookie);  in st_fdma_tx_status()
594 	if (fchan->fdesc && cookie == fchan->fdesc->vdesc.tx.cookie)  in st_fdma_tx_status()
595 		txstate->residue = st_fdma_desc_residue(fchan, vd, true);  in st_fdma_tx_status()
597 		txstate->residue = st_fdma_desc_residue(fchan, vd, false);  in st_fdma_tx_status()
601 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_tx_status()
608 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_issue_pending()  local
611 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_issue_pending()
613 	if (vchan_issue_pending(&fchan->vchan) && !fchan->fdesc)  in st_fdma_issue_pending()
614 		st_fdma_xfer_desc(fchan);  in st_fdma_issue_pending()
616 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_issue_pending()
622 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_pause()  local
623 	int ch_id = fchan->vchan.chan.chan_id;  in st_fdma_pause()
626 	dev_dbg(fchan->fdev->dev, "pause chan:%d\n", ch_id);  in st_fdma_pause()
628 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_pause()
629 	if (fchan->fdesc)  in st_fdma_pause()
630 		fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);  in st_fdma_pause()
631 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_pause()
640 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_resume()  local
641 	int ch_id = fchan->vchan.chan.chan_id;  in st_fdma_resume()
643 	dev_dbg(fchan->fdev->dev, "resume chan:%d\n", ch_id);  in st_fdma_resume()
645 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_resume()
646 	if (fchan->fdesc) {  in st_fdma_resume()
647 		val = fchan_read(fchan, FDMA_CH_CMD_OFST);  in st_fdma_resume()
649 		fchan_write(fchan, val, FDMA_CH_CMD_OFST);  in st_fdma_resume()
651 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_resume()
660 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_terminate_all()  local
661 	int ch_id = fchan->vchan.chan.chan_id;  in st_fdma_terminate_all()
664 	dev_dbg(fchan->fdev->dev, "terminate chan:%d\n", ch_id);  in st_fdma_terminate_all()
666 	spin_lock_irqsave(&fchan->vchan.lock, flags);  in st_fdma_terminate_all()
667 	fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);  in st_fdma_terminate_all()
668 	fchan->fdesc = NULL;  in st_fdma_terminate_all()
669 	vchan_get_all_descriptors(&fchan->vchan, &head);  in st_fdma_terminate_all()
670 	spin_unlock_irqrestore(&fchan->vchan.lock, flags);  in st_fdma_terminate_all()
671 	vchan_dma_desc_free_list(&fchan->vchan, &head);  in st_fdma_terminate_all()
679 	struct st_fdma_chan *fchan = to_st_fdma_chan(chan);  in st_fdma_slave_config()  local
681 	memcpy(&fchan->scfg, slave_cfg, sizeof(fchan->scfg));  in st_fdma_slave_config()
728 	struct st_fdma_chan *fchan;  in st_fdma_free()  local
732 		fchan = &fdev->chans[i];  in st_fdma_free()
733 		list_del(&fchan->vchan.chan.device_node);  in st_fdma_free()
734 		tasklet_kill(&fchan->vchan.task);  in st_fdma_free()
794 		struct st_fdma_chan *fchan = &fdev->chans[i];  in st_fdma_probe()  local
796 		fchan->fdev = fdev;  in st_fdma_probe()
797 		fchan->vchan.desc_free = st_fdma_free_desc;  in st_fdma_probe()
798 		vchan_init(&fchan->vchan, &fdev->dma_device);  in st_fdma_probe()