Lines Matching refs:atchan
80 static struct at_desc *atc_first_active(struct at_dma_chan *atchan) in atc_first_active() argument
82 return list_first_entry(&atchan->active_list, in atc_first_active()
86 static struct at_desc *atc_first_queued(struct at_dma_chan *atchan) in atc_first_queued() argument
88 return list_first_entry(&atchan->queue, in atc_first_queued()
126 static struct at_desc *atc_desc_get(struct at_dma_chan *atchan) in atc_desc_get() argument
133 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
134 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_desc_get()
141 dev_dbg(chan2dev(&atchan->chan_common), in atc_desc_get()
144 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
145 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_get()
150 ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC); in atc_desc_get()
152 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_get()
153 atchan->descs_allocated++; in atc_desc_get()
154 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_get()
156 dev_err(chan2dev(&atchan->chan_common), in atc_desc_get()
169 static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc) in atc_desc_put() argument
175 spin_lock_irqsave(&atchan->lock, flags); in atc_desc_put()
177 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
180 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_desc_put()
181 dev_vdbg(chan2dev(&atchan->chan_common), in atc_desc_put()
183 list_add(&desc->desc_node, &atchan->free_list); in atc_desc_put()
184 spin_unlock_irqrestore(&atchan->lock, flags); in atc_desc_put()
218 static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first) in atc_dostart() argument
220 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_dostart()
223 if (atc_chan_is_enabled(atchan)) { in atc_dostart()
224 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
226 dev_err(chan2dev(&atchan->chan_common), in atc_dostart()
228 channel_readl(atchan, SADDR), in atc_dostart()
229 channel_readl(atchan, DADDR), in atc_dostart()
230 channel_readl(atchan, CTRLA), in atc_dostart()
231 channel_readl(atchan, CTRLB), in atc_dostart()
232 channel_readl(atchan, DSCR)); in atc_dostart()
238 vdbg_dump_regs(atchan); in atc_dostart()
240 channel_writel(atchan, SADDR, 0); in atc_dostart()
241 channel_writel(atchan, DADDR, 0); in atc_dostart()
242 channel_writel(atchan, CTRLA, 0); in atc_dostart()
243 channel_writel(atchan, CTRLB, 0); in atc_dostart()
244 channel_writel(atchan, DSCR, first->txd.phys); in atc_dostart()
245 channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) | in atc_dostart()
247 channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) | in atc_dostart()
249 dma_writel(atdma, CHER, atchan->mask); in atc_dostart()
251 vdbg_dump_regs(atchan); in atc_dostart()
259 static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan, in atc_get_desc_by_cookie() argument
264 list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) { in atc_get_desc_by_cookie()
269 list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) { in atc_get_desc_by_cookie()
305 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_get_bytes_left() local
306 struct at_desc *desc_first = atc_first_active(atchan); in atc_get_bytes_left()
316 desc = atc_get_desc_by_cookie(atchan, cookie); in atc_get_bytes_left()
376 dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
378 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
383 new_dscr = channel_readl(atchan, DSCR); in atc_get_bytes_left()
403 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
427 ctrla = channel_readl(atchan, CTRLA); in atc_get_bytes_left()
441 atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc) in atc_chain_complete() argument
444 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_chain_complete()
446 dev_vdbg(chan2dev(&atchan->chan_common), in atc_chain_complete()
450 if (!atc_chan_is_cyclic(atchan)) in atc_chain_complete()
461 list_splice_init(&desc->tx_list, &atchan->free_list); in atc_chain_complete()
463 list_move(&desc->desc_node, &atchan->free_list); in atc_chain_complete()
468 if (!atc_chan_is_cyclic(atchan)) { in atc_chain_complete()
488 static void atc_complete_all(struct at_dma_chan *atchan) in atc_complete_all() argument
493 dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n"); in atc_complete_all()
499 if (!list_empty(&atchan->queue)) in atc_complete_all()
500 atc_dostart(atchan, atc_first_queued(atchan)); in atc_complete_all()
502 list_splice_init(&atchan->active_list, &list); in atc_complete_all()
504 list_splice_init(&atchan->queue, &atchan->active_list); in atc_complete_all()
507 atc_chain_complete(atchan, desc); in atc_complete_all()
516 static void atc_advance_work(struct at_dma_chan *atchan) in atc_advance_work() argument
518 dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n"); in atc_advance_work()
520 if (atc_chan_is_enabled(atchan)) in atc_advance_work()
523 if (list_empty(&atchan->active_list) || in atc_advance_work()
524 list_is_singular(&atchan->active_list)) { in atc_advance_work()
525 atc_complete_all(atchan); in atc_advance_work()
527 atc_chain_complete(atchan, atc_first_active(atchan)); in atc_advance_work()
529 atc_dostart(atchan, atc_first_active(atchan)); in atc_advance_work()
540 static void atc_handle_error(struct at_dma_chan *atchan) in atc_handle_error() argument
550 bad_desc = atc_first_active(atchan); in atc_handle_error()
555 list_splice_init(&atchan->queue, atchan->active_list.prev); in atc_handle_error()
558 if (!list_empty(&atchan->active_list)) in atc_handle_error()
559 atc_dostart(atchan, atc_first_active(atchan)); in atc_handle_error()
568 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
570 dev_crit(chan2dev(&atchan->chan_common), in atc_handle_error()
572 atc_dump_lli(atchan, &bad_desc->lli); in atc_handle_error()
574 atc_dump_lli(atchan, &child->lli); in atc_handle_error()
577 atc_chain_complete(atchan, bad_desc); in atc_handle_error()
586 static void atc_handle_cyclic(struct at_dma_chan *atchan) in atc_handle_cyclic() argument
588 struct at_desc *first = atc_first_active(atchan); in atc_handle_cyclic()
591 dev_vdbg(chan2dev(&atchan->chan_common), in atc_handle_cyclic()
593 channel_readl(atchan, DSCR)); in atc_handle_cyclic()
602 struct at_dma_chan *atchan = (struct at_dma_chan *)data; in atc_tasklet() local
605 spin_lock_irqsave(&atchan->lock, flags); in atc_tasklet()
606 if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status)) in atc_tasklet()
607 atc_handle_error(atchan); in atc_tasklet()
608 else if (atc_chan_is_cyclic(atchan)) in atc_tasklet()
609 atc_handle_cyclic(atchan); in atc_tasklet()
611 atc_advance_work(atchan); in atc_tasklet()
613 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tasklet()
619 struct at_dma_chan *atchan; in at_dma_interrupt() local
637 atchan = &atdma->chan[i]; in at_dma_interrupt()
642 AT_DMA_RES(i) | atchan->mask); in at_dma_interrupt()
644 set_bit(ATC_IS_ERROR, &atchan->status); in at_dma_interrupt()
646 tasklet_schedule(&atchan->tasklet); in at_dma_interrupt()
670 struct at_dma_chan *atchan = to_at_dma_chan(tx->chan); in atc_tx_submit() local
674 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_submit()
677 if (list_empty(&atchan->active_list)) { in atc_tx_submit()
680 atc_dostart(atchan, desc); in atc_tx_submit()
681 list_add_tail(&desc->desc_node, &atchan->active_list); in atc_tx_submit()
685 list_add_tail(&desc->desc_node, &atchan->queue); in atc_tx_submit()
688 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_submit()
704 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_interleaved() local
765 desc = atc_desc_get(atchan); in atc_prep_dma_interleaved()
804 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memcpy() local
841 desc = atc_desc_get(atchan); in atc_prep_dma_memcpy()
868 atc_desc_put(atchan, first); in atc_prep_dma_memcpy()
877 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_create_memset_desc() local
894 desc = atc_desc_get(atchan); in atc_create_memset_desc()
983 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_memset_sg() local
1050 atc_desc_put(atchan, first); in atc_prep_dma_memset_sg()
1068 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_slave_sg() local
1070 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_slave_sg()
1103 | ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if); in atc_prep_slave_sg()
1110 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1143 | ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if); in atc_prep_slave_sg()
1151 desc = atc_desc_get(atchan); in atc_prep_slave_sg()
1197 atc_desc_put(atchan, first); in atc_prep_slave_sg()
1231 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_dma_cyclic_fill_desc() local
1232 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_dma_cyclic_fill_desc()
1250 | ATC_SIF(atchan->mem_if) in atc_dma_cyclic_fill_desc()
1251 | ATC_DIF(atchan->per_if); in atc_dma_cyclic_fill_desc()
1262 | ATC_SIF(atchan->per_if) in atc_dma_cyclic_fill_desc()
1263 | ATC_DIF(atchan->mem_if); in atc_dma_cyclic_fill_desc()
1288 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_prep_dma_cyclic() local
1290 struct dma_slave_config *sconfig = &atchan->dma_sconfig; in atc_prep_dma_cyclic()
1308 was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1330 desc = atc_desc_get(atchan); in atc_prep_dma_cyclic()
1352 atc_desc_put(atchan, first); in atc_prep_dma_cyclic()
1354 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_prep_dma_cyclic()
1361 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_config() local
1369 memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig)); in atc_config()
1371 convert_burst(&atchan->dma_sconfig.src_maxburst); in atc_config()
1372 convert_burst(&atchan->dma_sconfig.dst_maxburst); in atc_config()
1379 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_pause() local
1381 int chan_id = atchan->chan_common.chan_id; in atc_pause()
1386 spin_lock_irqsave(&atchan->lock, flags); in atc_pause()
1389 set_bit(ATC_IS_PAUSED, &atchan->status); in atc_pause()
1391 spin_unlock_irqrestore(&atchan->lock, flags); in atc_pause()
1398 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_resume() local
1400 int chan_id = atchan->chan_common.chan_id; in atc_resume()
1405 if (!atc_chan_is_paused(atchan)) in atc_resume()
1408 spin_lock_irqsave(&atchan->lock, flags); in atc_resume()
1411 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_resume()
1413 spin_unlock_irqrestore(&atchan->lock, flags); in atc_resume()
1420 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_terminate_all() local
1422 int chan_id = atchan->chan_common.chan_id; in atc_terminate_all()
1436 spin_lock_irqsave(&atchan->lock, flags); in atc_terminate_all()
1439 dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask); in atc_terminate_all()
1442 while (dma_readl(atdma, CHSR) & atchan->mask) in atc_terminate_all()
1446 list_splice_init(&atchan->queue, &list); in atc_terminate_all()
1447 list_splice_init(&atchan->active_list, &list); in atc_terminate_all()
1451 atc_chain_complete(atchan, desc); in atc_terminate_all()
1453 clear_bit(ATC_IS_PAUSED, &atchan->status); in atc_terminate_all()
1455 clear_bit(ATC_IS_CYCLIC, &atchan->status); in atc_terminate_all()
1457 spin_unlock_irqrestore(&atchan->lock, flags); in atc_terminate_all()
1477 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_tx_status() local
1492 spin_lock_irqsave(&atchan->lock, flags); in atc_tx_status()
1497 spin_unlock_irqrestore(&atchan->lock, flags); in atc_tx_status()
1518 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_issue_pending() local
1524 if (atc_chan_is_cyclic(atchan)) in atc_issue_pending()
1527 spin_lock_irqsave(&atchan->lock, flags); in atc_issue_pending()
1528 atc_advance_work(atchan); in atc_issue_pending()
1529 spin_unlock_irqrestore(&atchan->lock, flags); in atc_issue_pending()
1541 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_alloc_chan_resources() local
1553 if (atc_chan_is_enabled(atchan)) { in atc_alloc_chan_resources()
1575 if (!list_empty(&atchan->free_list)) in atc_alloc_chan_resources()
1576 return atchan->descs_allocated; in atc_alloc_chan_resources()
1589 spin_lock_irqsave(&atchan->lock, flags); in atc_alloc_chan_resources()
1590 atchan->descs_allocated = i; in atc_alloc_chan_resources()
1591 list_splice(&tmp_list, &atchan->free_list); in atc_alloc_chan_resources()
1593 spin_unlock_irqrestore(&atchan->lock, flags); in atc_alloc_chan_resources()
1596 channel_writel(atchan, CFG, cfg); in atc_alloc_chan_resources()
1600 atchan->descs_allocated); in atc_alloc_chan_resources()
1602 return atchan->descs_allocated; in atc_alloc_chan_resources()
1611 struct at_dma_chan *atchan = to_at_dma_chan(chan); in atc_free_chan_resources() local
1617 atchan->descs_allocated); in atc_free_chan_resources()
1620 BUG_ON(!list_empty(&atchan->active_list)); in atc_free_chan_resources()
1621 BUG_ON(!list_empty(&atchan->queue)); in atc_free_chan_resources()
1622 BUG_ON(atc_chan_is_enabled(atchan)); in atc_free_chan_resources()
1624 list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) { in atc_free_chan_resources()
1630 list_splice_init(&atchan->free_list, &list); in atc_free_chan_resources()
1631 atchan->descs_allocated = 0; in atc_free_chan_resources()
1632 atchan->status = 0; in atc_free_chan_resources()
1660 struct at_dma_chan *atchan; in at_dma_xlate() local
1708 atchan = to_at_dma_chan(chan); in at_dma_xlate()
1709 atchan->per_if = dma_spec->args[0] & 0xff; in at_dma_xlate()
1710 atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff; in at_dma_xlate()
1888 struct at_dma_chan *atchan = &atdma->chan[i]; in at_dma_probe() local
1890 atchan->mem_if = AT_DMA_MEM_IF; in at_dma_probe()
1891 atchan->per_if = AT_DMA_PER_IF; in at_dma_probe()
1892 atchan->chan_common.device = &atdma->dma_common; in at_dma_probe()
1893 dma_cookie_init(&atchan->chan_common); in at_dma_probe()
1894 list_add_tail(&atchan->chan_common.device_node, in at_dma_probe()
1897 atchan->ch_regs = atdma->regs + ch_regs(i); in at_dma_probe()
1898 spin_lock_init(&atchan->lock); in at_dma_probe()
1899 atchan->mask = 1 << i; in at_dma_probe()
1901 INIT_LIST_HEAD(&atchan->active_list); in at_dma_probe()
1902 INIT_LIST_HEAD(&atchan->queue); in at_dma_probe()
1903 INIT_LIST_HEAD(&atchan->free_list); in at_dma_probe()
1905 tasklet_init(&atchan->tasklet, atc_tasklet, in at_dma_probe()
1906 (unsigned long)atchan); in at_dma_probe()
2009 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_remove() local
2014 tasklet_kill(&atchan->tasklet); in at_dma_remove()
2047 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_prepare() local
2049 if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan)) in at_dma_prepare()
2055 static void atc_suspend_cyclic(struct at_dma_chan *atchan) in atc_suspend_cyclic() argument
2057 struct dma_chan *chan = &atchan->chan_common; in atc_suspend_cyclic()
2061 if (!atc_chan_is_paused(atchan)) { in atc_suspend_cyclic()
2069 atchan->save_dscr = channel_readl(atchan, DSCR); in atc_suspend_cyclic()
2071 vdbg_dump_regs(atchan); in atc_suspend_cyclic()
2082 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_suspend_noirq() local
2084 if (atc_chan_is_cyclic(atchan)) in at_dma_suspend_noirq()
2085 atc_suspend_cyclic(atchan); in at_dma_suspend_noirq()
2086 atchan->save_cfg = channel_readl(atchan, CFG); in at_dma_suspend_noirq()
2096 static void atc_resume_cyclic(struct at_dma_chan *atchan) in atc_resume_cyclic() argument
2098 struct at_dma *atdma = to_at_dma(atchan->chan_common.device); in atc_resume_cyclic()
2102 channel_writel(atchan, SADDR, 0); in atc_resume_cyclic()
2103 channel_writel(atchan, DADDR, 0); in atc_resume_cyclic()
2104 channel_writel(atchan, CTRLA, 0); in atc_resume_cyclic()
2105 channel_writel(atchan, CTRLB, 0); in atc_resume_cyclic()
2106 channel_writel(atchan, DSCR, atchan->save_dscr); in atc_resume_cyclic()
2107 dma_writel(atdma, CHER, atchan->mask); in atc_resume_cyclic()
2112 vdbg_dump_regs(atchan); in atc_resume_cyclic()
2132 struct at_dma_chan *atchan = to_at_dma_chan(chan); in at_dma_resume_noirq() local
2134 channel_writel(atchan, CFG, atchan->save_cfg); in at_dma_resume_noirq()
2135 if (atc_chan_is_cyclic(atchan)) in at_dma_resume_noirq()
2136 atc_resume_cyclic(atchan); in at_dma_resume_noirq()