Lines Matching refs:atchan
85 static struct at_desc *atc_first_active(struct at_dma_chan *atchan) in atc_first_active() argument
87 return list_first_entry(&atchan->active_list, in atc_first_active()
91 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) in atc_first_queued() argument
93 return list_first_entry(&atchan->queue, in atc_first_queued()
131 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) in atc_desc_get() argument
139 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
140 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_desc_get()
147 dev_dbg(chan2dev(&atchan->chan_common), in atc_desc_get()
150 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
151 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_get()
156 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); in atc_desc_get()
158 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
159 atchan->descs_allocated++; in atc_desc_get()
160 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
162 dev_err(chan2dev(&atchan->chan_common), in atc_desc_get()
175 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) in atc_desc_put() argument
181 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_put()
183 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
186 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_desc_put()
187 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
189 list_add(&desc->desc_node, &atchan->free_list); in atc_desc_put()
190 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_put()
224 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) in atc_dostart() argument
226 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_dostart()
229 if (atc_chan_is_enabled(atchan)) { in atc_dostart()
230 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
232 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
234 channel_readl(atchan, SADDR), in atc_dostart()
235 channel_readl(atchan, DADDR), in atc_dostart()
236 channel_readl(atchan, CTRLA), in atc_dostart()
237 channel_readl(atchan, CTRLB), in atc_dostart()
238 channel_readl(atchan, DSCR)); in atc_dostart()
244 vdbg_dump_regs(atchan); in atc_dostart()
246 channel_writel(atchan, SADDR, 0); in atc_dostart()
247 channel_writel(atchan, DADDR, 0); in atc_dostart()
248 channel_writel(atchan, CTRLA, 0); in atc_dostart()
249 channel_writel(atchan, CTRLB, 0); in atc_dostart()
250 channel_writel(atchan, DSCR, first->txd.phys); in atc_dostart()
251 channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | in atc_dostart()
253 channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | in atc_dostart()
255 dma_writel(atdma, CHER, atchan->mask); in atc_dostart()
257 vdbg_dump_regs(atchan); in atc_dostart()
265 static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, in atc_get_desc_by_cookie() argument
270 list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { in atc_get_desc_by_cookie()
275 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { in atc_get_desc_by_cookie()
311 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_get_bytes_left() local
312 struct at_desc *desc_first = atc_first_active(atchan); in atc_get_bytes_left()
322 desc = atc_get_desc_by_cookie(atchan, cookie); in atc_get_bytes_left()
382 dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
384 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
389 new_dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
409 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
433 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
447 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) in atc_chain_complete() argument
450 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_chain_complete()
452 dev_vdbg(chan2dev(&atchan->chan_common), in atc_chain_complete()
456 if (!atc_chan_is_cyclic(atchan)) in atc_chain_complete()
467 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_chain_complete()
469 list_move(&desc->desc_node, &atchan->free_list); in atc_chain_complete()
474 if (!atc_chan_is_cyclic(atchan)) { in atc_chain_complete()
494 static void atc_complete_all(struct at_dma_chan *atchan) in atc_complete_all() argument
499 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); in atc_complete_all()
505 if (!list_empty(&atchan->queue)) in atc_complete_all()
506 atc_dostart(atchan, atc_first_queued(atchan)); in atc_complete_all()
508 list_splice_init(&atchan->active_list, &list); in atc_complete_all()
510 list_splice_init(&atchan->queue, &atchan->active_list); in atc_complete_all()
513 atc_chain_complete(atchan, desc); in atc_complete_all()
522 static void atc_advance_work(struct at_dma_chan *atchan) in atc_advance_work() argument
524 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); in atc_advance_work()
526 if (atc_chan_is_enabled(atchan)) in atc_advance_work()
529 if (list_empty(&atchan->active_list) || in atc_advance_work()
530 list_is_singular(&atchan->active_list)) { in atc_advance_work()
531 atc_complete_all(atchan); in atc_advance_work()
533 atc_chain_complete(atchan, atc_first_active(atchan)); in atc_advance_work()
535 atc_dostart(atchan, atc_first_active(atchan)); in atc_advance_work()
546 static void atc_handle_error(struct at_dma_chan *atchan) in atc_handle_error() argument
556 bad_desc = atc_first_active(atchan); in atc_handle_error()
561 list_splice_init(&atchan->queue, atchan->active_list.prev); in atc_handle_error()
564 if (!list_empty(&atchan->active_list)) in atc_handle_error()
565 atc_dostart(atchan, atc_first_active(atchan)); in atc_handle_error()
574 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
576 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
578 atc_dump_lli(atchan, &bad_desc->lli); in atc_handle_error()
580 atc_dump_lli(atchan, &child->lli); in atc_handle_error()
583 atc_chain_complete(atchan, bad_desc); in atc_handle_error()
592 static void atc_handle_cyclic(struct at_dma_chan *atchan) in atc_handle_cyclic() argument
594 struct at_desc *first = atc_first_active(atchan); in atc_handle_cyclic()
597 dev_vdbg(chan2dev(&atchan->chan_common), in atc_handle_cyclic()
599 channel_readl(atchan, DSCR)); in atc_handle_cyclic()
608 struct at_dma_chan *atchan = (struct at_dma_chan *)data; in atc_tasklet() local
611 spin_lock_irqsave(&atchan->lock, flags); in atc_tasklet()
612 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) in atc_tasklet()
613 atc_handle_error(atchan); in atc_tasklet()
614 else if (atc_chan_is_cyclic(atchan)) in atc_tasklet()
615 atc_handle_cyclic(atchan); in atc_tasklet()
617 atc_advance_work(atchan); in atc_tasklet()
619 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tasklet()
625 struct at_dma_chan *atchan; in at_dma_interrupt() local
643 atchan = &atdma->chan[i]; in at_dma_interrupt()
648 AT_DMA_RES(i) | atchan->mask); in at_dma_interrupt()
650 set_bit(ATC_IS_ERROR, &atchan->status); in at_dma_interrupt()
652 tasklet_schedule(&atchan->tasklet); in at_dma_interrupt()
676 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); in atc_tx_submit() local
680 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_submit()
683 if (list_empty(&atchan->active_list)) { in atc_tx_submit()
686 atc_dostart(atchan, desc); in atc_tx_submit()
687 list_add_tail(&desc->desc_node, &atchan->active_list); in atc_tx_submit()
691 list_add_tail(&desc->desc_node, &atchan->queue); in atc_tx_submit()
694 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_submit()
710 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_interleaved() local
771 desc = atc_desc_get(atchan); in atc_prep_dma_interleaved()
810 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memcpy() local
847 desc = atc_desc_get(atchan); in atc_prep_dma_memcpy()
874 atc_desc_put(atchan, first); in atc_prep_dma_memcpy()
883 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_create_memset_desc() local
900 desc = atc_desc_get(atchan); in atc_create_memset_desc()
989 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memset_sg() local
1056 atc_desc_put(atchan, first); in atc_prep_dma_memset_sg()
1074 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_slave_sg() local
1076 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_slave_sg()
1109 | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); in atc_prep_slave_sg()
1116 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1149 | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); in atc_prep_slave_sg()
1157 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1203 atc_desc_put(atchan, first); in atc_prep_slave_sg()
1237 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_dma_cyclic_fill_desc() local
1238 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_dma_cyclic_fill_desc()
1256 | ATC_SIF(atchan->mem_if) in atc_dma_cyclic_fill_desc()
1257 | ATC_DIF(atchan->per_if); in atc_dma_cyclic_fill_desc()
1268 | ATC_SIF(atchan->per_if) in atc_dma_cyclic_fill_desc()
1269 | ATC_DIF(atchan->mem_if); in atc_dma_cyclic_fill_desc()
1294 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_cyclic() local
1296 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_dma_cyclic()
1314 was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1336 desc = atc_desc_get(atchan); in atc_prep_dma_cyclic()
1358 atc_desc_put(atchan, first); in atc_prep_dma_cyclic()
1360 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1367 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_config() local
1375 memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); in atc_config()
1377 convert_burst(&atchan->dma_sconfig.src_maxburst); in atc_config()
1378 convert_burst(&atchan->dma_sconfig.dst_maxburst); in atc_config()
1385 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_pause() local
1387 int chan_id = atchan->chan_common.chan_id; in atc_pause()
1394 spin_lock_irqsave(&atchan->lock, flags); in atc_pause()
1397 set_bit(ATC_IS_PAUSED, &atchan->status); in atc_pause()
1399 spin_unlock_irqrestore(&atchan->lock, flags); in atc_pause()
1406 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_resume() local
1408 int chan_id = atchan->chan_common.chan_id; in atc_resume()
1415 if (!atc_chan_is_paused(atchan)) in atc_resume()
1418 spin_lock_irqsave(&atchan->lock, flags); in atc_resume()
1421 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_resume()
1423 spin_unlock_irqrestore(&atchan->lock, flags); in atc_resume()
1430 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_terminate_all() local
1432 int chan_id = atchan->chan_common.chan_id; in atc_terminate_all()
1446 spin_lock_irqsave(&atchan->lock, flags); in atc_terminate_all()
1449 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); in atc_terminate_all()
1452 while (dma_readl(atdma, CHSR) & atchan->mask) in atc_terminate_all()
1456 list_splice_init(&atchan->queue, &list); in atc_terminate_all()
1457 list_splice_init(&atchan->active_list, &list); in atc_terminate_all()
1461 atc_chain_complete(atchan, desc); in atc_terminate_all()
1463 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_terminate_all()
1465 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_terminate_all()
1467 spin_unlock_irqrestore(&atchan->lock, flags); in atc_terminate_all()
1487 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_tx_status() local
1502 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_status()
1507 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_status()
1528 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_issue_pending() local
1534 if (atc_chan_is_cyclic(atchan)) in atc_issue_pending()
1537 spin_lock_irqsave(&atchan->lock, flags); in atc_issue_pending()
1538 atc_advance_work(atchan); in atc_issue_pending()
1539 spin_unlock_irqrestore(&atchan->lock, flags); in atc_issue_pending()
1551 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_alloc_chan_resources() local
1563 if (atc_chan_is_enabled(atchan)) { in atc_alloc_chan_resources()
1585 if (!list_empty(&atchan->free_list)) in atc_alloc_chan_resources()
1586 return atchan->descs_allocated; in atc_alloc_chan_resources()
1599 spin_lock_irqsave(&atchan->lock, flags); in atc_alloc_chan_resources()
1600 atchan->descs_allocated = i; in atc_alloc_chan_resources()
1601 list_splice(&tmp_list, &atchan->free_list); in atc_alloc_chan_resources()
1603 spin_unlock_irqrestore(&atchan->lock, flags); in atc_alloc_chan_resources()
1606 channel_writel(atchan, CFG, cfg); in atc_alloc_chan_resources()
1610 atchan->descs_allocated); in atc_alloc_chan_resources()
1612 return atchan->descs_allocated; in atc_alloc_chan_resources()
1621 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_free_chan_resources() local
1627 atchan->descs_allocated); in atc_free_chan_resources()
1630 BUG_ON(!list_empty(&atchan->active_list)); in atc_free_chan_resources()
1631 BUG_ON(!list_empty(&atchan->queue)); in atc_free_chan_resources()
1632 BUG_ON(atc_chan_is_enabled(atchan)); in atc_free_chan_resources()
1634 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_free_chan_resources()
1640 list_splice_init(&atchan->free_list, &list); in atc_free_chan_resources()
1641 atchan->descs_allocated = 0; in atc_free_chan_resources()
1642 atchan->status = 0; in atc_free_chan_resources()
1664 struct at_dma_chan *atchan; in at_dma_xlate() local
1712 atchan = to_at_dma_chan(chan); in at_dma_xlate()
1713 atchan->per_if = dma_spec->args[0] & 0xff; in at_dma_xlate()
1714 atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; in at_dma_xlate()
1892 struct at_dma_chan *atchan = &atdma->chan[i]; in at_dma_probe() local
1894 atchan->mem_if = AT_DMA_MEM_IF; in at_dma_probe()
1895 atchan->per_if = AT_DMA_PER_IF; in at_dma_probe()
1896 atchan->chan_common.device = &atdma->dma_common; in at_dma_probe()
1897 dma_cookie_init(&atchan->chan_common); in at_dma_probe()
1898 list_add_tail(&atchan->chan_common.device_node, in at_dma_probe()
1901 atchan->ch_regs = atdma->regs + ch_regs(i); in at_dma_probe()
1902 spin_lock_init(&atchan->lock); in at_dma_probe()
1903 atchan->mask = 1 << i; in at_dma_probe()
1905 INIT_LIST_HEAD(&atchan->active_list); in at_dma_probe()
1906 INIT_LIST_HEAD(&atchan->queue); in at_dma_probe()
1907 INIT_LIST_HEAD(&atchan->free_list); in at_dma_probe()
1909 tasklet_init(&atchan->tasklet, atc_tasklet, in at_dma_probe()
1910 (unsigned long)atchan); in at_dma_probe()
2011 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_remove() local
2016 tasklet_kill(&atchan->tasklet); in at_dma_remove()
2049 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_prepare() local
2051 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) in at_dma_prepare()
2057 static void atc_suspend_cyclic(struct at_dma_chan *atchan) in atc_suspend_cyclic() argument
2059 struct dma_chan *chan = &atchan->chan_common; in atc_suspend_cyclic()
2063 if (!atc_chan_is_paused(atchan)) { in atc_suspend_cyclic()
2071 atchan->save_dscr = channel_readl(atchan, DSCR); in atc_suspend_cyclic()
2073 vdbg_dump_regs(atchan); in atc_suspend_cyclic()
2084 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_suspend_noirq() local
2086 if (atc_chan_is_cyclic(atchan)) in at_dma_suspend_noirq()
2087 atc_suspend_cyclic(atchan); in at_dma_suspend_noirq()
2088 atchan->save_cfg = channel_readl(atchan, CFG); in at_dma_suspend_noirq()
2098 static void atc_resume_cyclic(struct at_dma_chan *atchan) in atc_resume_cyclic() argument
2100 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_resume_cyclic()
2104 channel_writel(atchan, SADDR, 0); in atc_resume_cyclic()
2105 channel_writel(atchan, DADDR, 0); in atc_resume_cyclic()
2106 channel_writel(atchan, CTRLA, 0); in atc_resume_cyclic()
2107 channel_writel(atchan, CTRLB, 0); in atc_resume_cyclic()
2108 channel_writel(atchan, DSCR, atchan->save_dscr); in atc_resume_cyclic()
2109 dma_writel(atdma, CHER, atchan->mask); in atc_resume_cyclic()
2114 vdbg_dump_regs(atchan); in atc_resume_cyclic()
2134 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_resume_noirq() local
2136 channel_writel(atchan, CFG, atchan->save_cfg); in at_dma_resume_noirq()
2137 if (atc_chan_is_cyclic(atchan)) in at_dma_resume_noirq()
2138 atc_resume_cyclic(atchan); in at_dma_resume_noirq()