Lines Matching full:ud
215 struct udma_dev *ud; member
335 static int navss_psil_pair(struct udma_dev *ud, u32 src_thread, u32 dst_thread) in navss_psil_pair() argument
337 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in navss_psil_pair()
345 static int navss_psil_unpair(struct udma_dev *ud, u32 src_thread, in navss_psil_unpair() argument
348 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in navss_psil_unpair()
365 struct device *dev = uc->ud->dev; in udma_dump_chan_stdata()
442 struct udma_dev *ud = uc->ud; in udma_free_hwdesc() local
444 dma_free_coherent(ud->dev, d->hwdesc[0].cppi5_desc_size, in udma_free_hwdesc()
454 struct udma_dev *ud = container_of(work, typeof(*ud), purge_work); in udma_purge_desc_work() local
459 spin_lock_irqsave(&ud->lock, flags); in udma_purge_desc_work()
460 list_splice_tail_init(&ud->desc_to_purge, &head); in udma_purge_desc_work()
461 spin_unlock_irqrestore(&ud->lock, flags); in udma_purge_desc_work()
473 if (!list_empty(&ud->desc_to_purge)) in udma_purge_desc_work()
474 schedule_work(&ud->purge_work); in udma_purge_desc_work()
479 struct udma_dev *ud = to_udma_dev(vd->tx.chan->device); in udma_desc_free() local
493 spin_lock_irqsave(&ud->lock, flags); in udma_desc_free()
494 list_add_tail(&vd->node, &ud->desc_to_purge); in udma_desc_free()
495 spin_unlock_irqrestore(&ud->lock, flags); in udma_desc_free()
497 schedule_work(&ud->purge_work); in udma_desc_free()
545 return uc->ud->rx_flush.hwdescs[uc->config.pkt_mode].cppi5_desc_paddr; in udma_get_rx_flush_hwdesc_paddr()
722 uc->ud->ddev.device_free_chan_resources(&uc->vc.chan); in udma_reset_chan()
726 ret = uc->ud->ddev.device_alloc_chan_resources(&uc->vc.chan); in udma_reset_chan()
805 uc->ud->match_data; in udma_start()
1052 dev_err(uc->ud->dev, "not matching descriptors!\n"); in udma_ring_irq_handler()
1113 * @ud: UDMA device
1127 static int __udma_alloc_gp_rflow_range(struct udma_dev *ud, int from, int cnt) in __udma_alloc_gp_rflow_range() argument
1134 tmp_from = ud->rchan_cnt; in __udma_alloc_gp_rflow_range()
1136 if (tmp_from < ud->rchan_cnt) in __udma_alloc_gp_rflow_range()
1139 if (tmp_from + cnt > ud->rflow_cnt) in __udma_alloc_gp_rflow_range()
1142 bitmap_or(tmp, ud->rflow_gp_map, ud->rflow_gp_map_allocated, in __udma_alloc_gp_rflow_range()
1143 ud->rflow_cnt); in __udma_alloc_gp_rflow_range()
1146 ud->rflow_cnt, in __udma_alloc_gp_rflow_range()
1148 if (start >= ud->rflow_cnt) in __udma_alloc_gp_rflow_range()
1154 bitmap_set(ud->rflow_gp_map_allocated, start, cnt); in __udma_alloc_gp_rflow_range()
1158 static int __udma_free_gp_rflow_range(struct udma_dev *ud, int from, int cnt) in __udma_free_gp_rflow_range() argument
1160 if (from < ud->rchan_cnt) in __udma_free_gp_rflow_range()
1162 if (from + cnt > ud->rflow_cnt) in __udma_free_gp_rflow_range()
1165 bitmap_clear(ud->rflow_gp_map_allocated, from, cnt); in __udma_free_gp_rflow_range()
1169 static struct udma_rflow *__udma_get_rflow(struct udma_dev *ud, int id) in __udma_get_rflow() argument
1178 if (id < 0 || id >= ud->rflow_cnt) in __udma_get_rflow()
1181 if (test_bit(id, ud->rflow_in_use)) in __udma_get_rflow()
1185 if (!test_bit(id, ud->rflow_gp_map) && in __udma_get_rflow()
1186 !test_bit(id, ud->rflow_gp_map_allocated)) in __udma_get_rflow()
1189 dev_dbg(ud->dev, "get rflow%d\n", id); in __udma_get_rflow()
1190 set_bit(id, ud->rflow_in_use); in __udma_get_rflow()
1191 return &ud->rflows[id]; in __udma_get_rflow()
1194 static void __udma_put_rflow(struct udma_dev *ud, struct udma_rflow *rflow) in __udma_put_rflow() argument
1196 if (!test_bit(rflow->id, ud->rflow_in_use)) { in __udma_put_rflow()
1197 dev_err(ud->dev, "attempt to put unused rflow%d\n", rflow->id); in __udma_put_rflow()
1201 dev_dbg(ud->dev, "put rflow%d\n", rflow->id); in __udma_put_rflow()
1202 clear_bit(rflow->id, ud->rflow_in_use); in __udma_put_rflow()
1206 static struct udma_##res *__udma_reserve_##res(struct udma_dev *ud, \
1211 if (test_bit(id, ud->res##_map)) { \
1212 dev_err(ud->dev, "res##%d is in use\n", id); \
1218 if (tpl >= ud->tpl_levels) \
1219 tpl = ud->tpl_levels - 1; \
1221 start = ud->tpl_start_idx[tpl]; \
1223 id = find_next_zero_bit(ud->res##_map, ud->res##_cnt, \
1225 if (id == ud->res##_cnt) { \
1230 set_bit(id, ud->res##_map); \
1231 return &ud->res##s[id]; \
1239 struct udma_dev *ud = uc->ud; in udma_get_tchan() local
1242 dev_dbg(ud->dev, "chan%d: already have tchan%d allocated\n", in udma_get_tchan()
1247 uc->tchan = __udma_reserve_tchan(ud, uc->config.channel_tpl, -1); in udma_get_tchan()
1254 struct udma_dev *ud = uc->ud; in udma_get_rchan() local
1257 dev_dbg(ud->dev, "chan%d: already have rchan%d allocated\n", in udma_get_rchan()
1262 uc->rchan = __udma_reserve_rchan(ud, uc->config.channel_tpl, -1); in udma_get_rchan()
1269 struct udma_dev *ud = uc->ud; in udma_get_chan_pair() local
1273 dev_info(ud->dev, "chan%d: already have %d pair allocated\n", in udma_get_chan_pair()
1279 dev_err(ud->dev, "chan%d: already have tchan%d allocated\n", in udma_get_chan_pair()
1283 dev_err(ud->dev, "chan%d: already have rchan%d allocated\n", in udma_get_chan_pair()
1289 end = min(ud->tchan_cnt, ud->rchan_cnt); in udma_get_chan_pair()
1291 chan_id = ud->tpl_start_idx[ud->tpl_levels - 1]; in udma_get_chan_pair()
1293 if (!test_bit(chan_id, ud->tchan_map) && in udma_get_chan_pair()
1294 !test_bit(chan_id, ud->rchan_map)) in udma_get_chan_pair()
1301 set_bit(chan_id, ud->tchan_map); in udma_get_chan_pair()
1302 set_bit(chan_id, ud->rchan_map); in udma_get_chan_pair()
1303 uc->tchan = &ud->tchans[chan_id]; in udma_get_chan_pair()
1304 uc->rchan = &ud->rchans[chan_id]; in udma_get_chan_pair()
1311 struct udma_dev *ud = uc->ud; in udma_get_rflow() local
1314 dev_err(ud->dev, "chan%d: does not have rchan??\n", uc->id); in udma_get_rflow()
1319 dev_dbg(ud->dev, "chan%d: already have rflow%d allocated\n", in udma_get_rflow()
1324 uc->rflow = __udma_get_rflow(ud, flow_id); in udma_get_rflow()
1331 struct udma_dev *ud = uc->ud; in udma_put_rchan() local
1334 dev_dbg(ud->dev, "chan%d: put rchan%d\n", uc->id, in udma_put_rchan()
1336 clear_bit(uc->rchan->id, ud->rchan_map); in udma_put_rchan()
1343 struct udma_dev *ud = uc->ud; in udma_put_tchan() local
1346 dev_dbg(ud->dev, "chan%d: put tchan%d\n", uc->id, in udma_put_tchan()
1348 clear_bit(uc->tchan->id, ud->tchan_map); in udma_put_tchan()
1355 struct udma_dev *ud = uc->ud; in udma_put_rflow() local
1358 dev_dbg(ud->dev, "chan%d: put rflow%d\n", uc->id, in udma_put_rflow()
1360 __udma_put_rflow(ud, uc->rflow); in udma_put_rflow()
1381 struct udma_dev *ud = uc->ud; in udma_alloc_tx_resources() local
1388 ret = k3_ringacc_request_rings_pair(ud->ringacc, uc->tchan->id, -1, in udma_alloc_tx_resources()
1441 struct udma_dev *ud = uc->ud; in udma_alloc_rx_resources() local
1462 fd_ring_id = ud->tchan_cnt + ud->echan_cnt + uc->rchan->id; in udma_alloc_rx_resources()
1463 ret = k3_ringacc_request_rings_pair(ud->ringacc, fd_ring_id, -1, in udma_alloc_rx_resources()
1525 struct udma_dev *ud = uc->ud; in udma_tisci_m2m_channel_config() local
1526 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in udma_tisci_m2m_channel_config()
1543 req_tx.tx_atype = ud->atype; in udma_tisci_m2m_channel_config()
1547 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); in udma_tisci_m2m_channel_config()
1557 req_rx.rx_atype = ud->atype; in udma_tisci_m2m_channel_config()
1561 dev_err(ud->dev, "rchan%d alloc failed %d\n", rchan->id, ret); in udma_tisci_m2m_channel_config()
1568 struct udma_dev *ud = uc->ud; in udma_tisci_tx_channel_config() local
1569 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in udma_tisci_tx_channel_config()
1597 dev_err(ud->dev, "tchan%d cfg failed %d\n", tchan->id, ret); in udma_tisci_tx_channel_config()
1604 struct udma_dev *ud = uc->ud; in udma_tisci_rx_channel_config() local
1605 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in udma_tisci_rx_channel_config()
1634 dev_err(ud->dev, "rchan%d cfg failed %d\n", rchan->id, ret); in udma_tisci_rx_channel_config()
1678 dev_err(ud->dev, "flow%d config failed: %d\n", rchan->id, ret); in udma_tisci_rx_channel_config()
1686 struct udma_dev *ud = to_udma_dev(chan->device); in udma_alloc_chan_resources() local
1687 const struct udma_soc_data *soc_data = ud->soc_data; in udma_alloc_chan_resources()
1703 uc->hdesc_pool = dma_pool_create(uc->name, ud->ddev.dev, in udma_alloc_chan_resources()
1705 ud->desc_align, in udma_alloc_chan_resources()
1708 dev_err(ud->ddev.dev, in udma_alloc_chan_resources()
1727 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-MEM\n", __func__, in udma_alloc_chan_resources()
1746 uc->config.src_thread = ud->psil_base + uc->tchan->id; in udma_alloc_chan_resources()
1747 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | in udma_alloc_chan_resources()
1757 dev_dbg(uc->ud->dev, "%s: chan%d as MEM-to-DEV\n", __func__, in udma_alloc_chan_resources()
1764 uc->config.src_thread = ud->psil_base + uc->tchan->id; in udma_alloc_chan_resources()
1775 dev_dbg(uc->ud->dev, "%s: chan%d as DEV-to-MEM\n", __func__, in udma_alloc_chan_resources()
1783 uc->config.dst_thread = (ud->psil_base + uc->rchan->id) | in udma_alloc_chan_resources()
1793 dev_err(uc->ud->dev, "%s: chan%d invalid direction (%u)\n", in udma_alloc_chan_resources()
1805 dev_warn(ud->dev, "chan%d: is running!\n", uc->id); in udma_alloc_chan_resources()
1808 dev_err(ud->dev, "chan%d: won't stop!\n", uc->id); in udma_alloc_chan_resources()
1815 ret = navss_psil_pair(ud, uc->config.src_thread, uc->config.dst_thread); in udma_alloc_chan_resources()
1817 dev_err(ud->dev, "PSI-L pairing failed: 0x%04x -> 0x%04x\n", in udma_alloc_chan_resources()
1826 dev_err(ud->dev, "Failed to get ring irq (index: %u)\n", in udma_alloc_chan_resources()
1835 dev_err(ud->dev, "chan%d: ring irq request failed\n", uc->id); in udma_alloc_chan_resources()
1841 uc->irq_num_udma = ti_sci_inta_msi_get_virq(ud->dev, in udma_alloc_chan_resources()
1844 dev_err(ud->dev, "Failed to get udma irq (index: %u)\n", in udma_alloc_chan_resources()
1854 dev_err(ud->dev, "chan%d: UDMA irq request failed\n", in udma_alloc_chan_resources()
1871 navss_psil_unpair(ud, uc->config.src_thread, uc->config.dst_thread); in udma_alloc_chan_resources()
1914 dev_err(uc->ud->dev, "Unsupported TR size of %zu\n", tr_size); in udma_alloc_tr_desc()
1938 uc->ud->desc_align); in udma_alloc_tr_desc()
1939 hwdesc->cppi5_desc_vaddr = dma_alloc_coherent(uc->ud->dev, in udma_alloc_tr_desc()
2055 dev_err(uc->ud->dev, "size %u is not supported\n", in udma_prep_slave_sg_tr()
2140 d->static_tr.bstcnt > uc->ud->match_data->statictr_z_mask) in udma_configure_statictr()
2182 dev_err(uc->ud->dev, in udma_prep_slave_sg_pkt()
2218 dev_err(uc->ud->dev, in udma_prep_slave_sg_pkt()
2375 dev_err(uc->ud->dev, in udma_prep_slave_sg()
2406 dev_err(uc->ud->dev, "size %zu is not supported\n", in udma_prep_dma_cyclic_tr()
2492 dev_err(uc->ud->dev, in udma_prep_dma_cyclic_pkt()
2548 dev_err(uc->ud->dev, "%s: bad direction?\n", __func__); in udma_prep_dma_cyclic()
2572 dev_err(uc->ud->dev, in udma_prep_dma_cyclic()
2610 dev_err(uc->ud->dev, "size %zu is not supported\n", in udma_prep_dma_memcpy()
2859 dev_warn(uc->ud->dev, "chan%d teardown timeout!\n", in udma_synchronize()
2868 dev_warn(uc->ud->dev, "chan%d refused to stop!\n", uc->id); in udma_synchronize()
2950 struct udma_dev *ud = to_udma_dev(chan->device); in udma_free_chan_resources() local
2973 navss_psil_unpair(ud, uc->config.src_thread, in udma_free_chan_resources()
3004 struct udma_dev *ud; in udma_dma_filter_fn() local
3011 ud = uc->ud; in udma_dma_filter_fn()
3015 dev_err(ud->dev, "Invalid channel atype: %u\n", in udma_dma_filter_fn()
3030 dev_err(ud->dev, "No configuration for psi-l thread 0x%04x\n", in udma_dma_filter_fn()
3044 const struct udma_match_data *match_data = ud->match_data; in udma_dma_filter_fn()
3060 ucc->metadata_size, ud->desc_align); in udma_dma_filter_fn()
3062 dev_dbg(ud->dev, "chan%d: Remote thread: 0x%04x (%s)\n", uc->id, in udma_dma_filter_fn()
3071 struct udma_dev *ud = ofdma->of_dma_data; in udma_of_xlate() local
3072 dma_cap_mask_t mask = ud->ddev.cap_mask; in udma_of_xlate()
3088 dev_err(ud->dev, "get channel fail in %s.\n", __func__); in udma_of_xlate()
3158 static int udma_get_mmrs(struct platform_device *pdev, struct udma_dev *ud) in udma_get_mmrs() argument
3163 ud->mmrs[i] = devm_platform_ioremap_resource_byname(pdev, mmr_names[i]); in udma_get_mmrs()
3164 if (IS_ERR(ud->mmrs[i])) in udma_get_mmrs()
3165 return PTR_ERR(ud->mmrs[i]); in udma_get_mmrs()
3171 static int udma_setup_resources(struct udma_dev *ud) in udma_setup_resources() argument
3173 struct device *dev = ud->dev; in udma_setup_resources()
3178 struct udma_tisci_rm *tisci_rm = &ud->tisci_rm; in udma_setup_resources()
3183 cap2 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(2)); in udma_setup_resources()
3184 cap3 = udma_read(ud->mmrs[MMR_GCFG], UDMA_CAP_REG(3)); in udma_setup_resources()
3186 ud->rflow_cnt = UDMA_CAP3_RFLOW_CNT(cap3); in udma_setup_resources()
3187 ud->tchan_cnt = UDMA_CAP2_TCHAN_CNT(cap2); in udma_setup_resources()
3188 ud->echan_cnt = UDMA_CAP2_ECHAN_CNT(cap2); in udma_setup_resources()
3189 ud->rchan_cnt = UDMA_CAP2_RCHAN_CNT(cap2); in udma_setup_resources()
3190 ch_count = ud->tchan_cnt + ud->rchan_cnt; in udma_setup_resources()
3195 ud->tpl_levels = 2; in udma_setup_resources()
3196 ud->tpl_start_idx[0] = 8; in udma_setup_resources()
3199 ud->tpl_levels = 2; in udma_setup_resources()
3200 ud->tpl_start_idx[0] = 2; in udma_setup_resources()
3202 ud->tpl_levels = 3; in udma_setup_resources()
3203 ud->tpl_start_idx[1] = UDMA_CAP3_UCHAN_CNT(cap3); in udma_setup_resources()
3204 ud->tpl_start_idx[0] = ud->tpl_start_idx[1] + in udma_setup_resources()
3207 ud->tpl_levels = 2; in udma_setup_resources()
3208 ud->tpl_start_idx[0] = UDMA_CAP3_HCHAN_CNT(cap3); in udma_setup_resources()
3210 ud->tpl_levels = 1; in udma_setup_resources()
3213 ud->tchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->tchan_cnt), in udma_setup_resources()
3215 ud->tchans = devm_kcalloc(dev, ud->tchan_cnt, sizeof(*ud->tchans), in udma_setup_resources()
3217 ud->rchan_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rchan_cnt), in udma_setup_resources()
3219 ud->rchans = devm_kcalloc(dev, ud->rchan_cnt, sizeof(*ud->rchans), in udma_setup_resources()
3221 ud->rflow_gp_map = devm_kmalloc_array(dev, BITS_TO_LONGS(ud->rflow_cnt), in udma_setup_resources()
3224 ud->rflow_gp_map_allocated = devm_kcalloc(dev, in udma_setup_resources()
3225 BITS_TO_LONGS(ud->rflow_cnt), in udma_setup_resources()
3228 ud->rflow_in_use = devm_kcalloc(dev, BITS_TO_LONGS(ud->rflow_cnt), in udma_setup_resources()
3231 ud->rflows = devm_kcalloc(dev, ud->rflow_cnt, sizeof(*ud->rflows), in udma_setup_resources()
3234 if (!ud->tchan_map || !ud->rchan_map || !ud->rflow_gp_map || in udma_setup_resources()
3235 !ud->rflow_gp_map_allocated || !ud->tchans || !ud->rchans || in udma_setup_resources()
3236 !ud->rflows || !ud->rflow_in_use) in udma_setup_resources()
3244 bitmap_set(ud->rflow_gp_map_allocated, 0, ud->rchan_cnt); in udma_setup_resources()
3247 bitmap_set(ud->rflow_gp_map, 0, ud->rflow_cnt); in udma_setup_resources()
3259 bitmap_zero(ud->tchan_map, ud->tchan_cnt); in udma_setup_resources()
3261 bitmap_fill(ud->tchan_map, ud->tchan_cnt); in udma_setup_resources()
3264 bitmap_clear(ud->tchan_map, rm_desc->start, in udma_setup_resources()
3275 bitmap_zero(ud->rchan_map, ud->rchan_cnt); in udma_setup_resources()
3277 bitmap_fill(ud->rchan_map, ud->rchan_cnt); in udma_setup_resources()
3280 bitmap_clear(ud->rchan_map, rm_desc->start, in udma_setup_resources()
3297 ud->soc_data->rchan_oes_offset; in udma_setup_resources()
3300 ret = ti_sci_inta_msi_domain_alloc_irqs(ud->dev, &irq_res); in udma_setup_resources()
3303 dev_err(ud->dev, "Failed to allocate MSI interrupts\n"); in udma_setup_resources()
3311 bitmap_clear(ud->rflow_gp_map, ud->rchan_cnt, in udma_setup_resources()
3312 ud->rflow_cnt - ud->rchan_cnt); in udma_setup_resources()
3316 bitmap_clear(ud->rflow_gp_map, rm_desc->start, in udma_setup_resources()
3323 ch_count -= bitmap_weight(ud->tchan_map, ud->tchan_cnt); in udma_setup_resources()
3324 ch_count -= bitmap_weight(ud->rchan_map, ud->rchan_cnt); in udma_setup_resources()
3328 ud->channels = devm_kcalloc(dev, ch_count, sizeof(*ud->channels), in udma_setup_resources()
3330 if (!ud->channels) in udma_setup_resources()
3335 ud->tchan_cnt - bitmap_weight(ud->tchan_map, ud->tchan_cnt), in udma_setup_resources()
3336 ud->rchan_cnt - bitmap_weight(ud->rchan_map, ud->rchan_cnt), in udma_setup_resources()
3337 ud->rflow_cnt - bitmap_weight(ud->rflow_gp_map, in udma_setup_resources()
3338 ud->rflow_cnt)); in udma_setup_resources()
3343 static int udma_setup_rx_flush(struct udma_dev *ud) in udma_setup_rx_flush() argument
3345 struct udma_rx_flush *rx_flush = &ud->rx_flush; in udma_setup_rx_flush()
3349 struct device *dev = ud->dev; in udma_setup_rx_flush()
3371 ud->desc_align); in udma_setup_rx_flush()
3411 ud->desc_align); in udma_setup_rx_flush()
3509 struct udma_dev *ud; in udma_probe() local
3518 ud = devm_kzalloc(dev, sizeof(*ud), GFP_KERNEL); in udma_probe()
3519 if (!ud) in udma_probe()
3522 ret = udma_get_mmrs(pdev, ud); in udma_probe()
3526 ud->tisci_rm.tisci = ti_sci_get_by_phandle(dev->of_node, "ti,sci"); in udma_probe()
3527 if (IS_ERR(ud->tisci_rm.tisci)) in udma_probe()
3528 return PTR_ERR(ud->tisci_rm.tisci); in udma_probe()
3531 &ud->tisci_rm.tisci_dev_id); in udma_probe()
3536 pdev->id = ud->tisci_rm.tisci_dev_id; in udma_probe()
3539 &ud->tisci_rm.tisci_navss_dev_id); in udma_probe()
3545 ret = of_property_read_u32(dev->of_node, "ti,udma-atype", &ud->atype); in udma_probe()
3546 if (!ret && ud->atype > 2) { in udma_probe()
3547 dev_err(dev, "Invalid atype: %u\n", ud->atype); in udma_probe()
3551 ud->tisci_rm.tisci_udmap_ops = &ud->tisci_rm.tisci->ops.rm_udmap_ops; in udma_probe()
3552 ud->tisci_rm.tisci_psil_ops = &ud->tisci_rm.tisci->ops.rm_psil_ops; in udma_probe()
3554 ud->ringacc = of_k3_ringacc_get_by_phandle(dev->of_node, "ti,ringacc"); in udma_probe()
3555 if (IS_ERR(ud->ringacc)) in udma_probe()
3556 return PTR_ERR(ud->ringacc); in udma_probe()
3570 ud->match_data = match->data; in udma_probe()
3577 ud->soc_data = soc->data; in udma_probe()
3579 dma_cap_set(DMA_SLAVE, ud->ddev.cap_mask); in udma_probe()
3580 dma_cap_set(DMA_CYCLIC, ud->ddev.cap_mask); in udma_probe()
3582 ud->ddev.device_alloc_chan_resources = udma_alloc_chan_resources; in udma_probe()
3583 ud->ddev.device_config = udma_slave_config; in udma_probe()
3584 ud->ddev.device_prep_slave_sg = udma_prep_slave_sg; in udma_probe()
3585 ud->ddev.device_prep_dma_cyclic = udma_prep_dma_cyclic; in udma_probe()
3586 ud->ddev.device_issue_pending = udma_issue_pending; in udma_probe()
3587 ud->ddev.device_tx_status = udma_tx_status; in udma_probe()
3588 ud->ddev.device_pause = udma_pause; in udma_probe()
3589 ud->ddev.device_resume = udma_resume; in udma_probe()
3590 ud->ddev.device_terminate_all = udma_terminate_all; in udma_probe()
3591 ud->ddev.device_synchronize = udma_synchronize; in udma_probe()
3593 ud->ddev.dbg_summary_show = udma_dbg_summary_show; in udma_probe()
3596 ud->ddev.device_free_chan_resources = udma_free_chan_resources; in udma_probe()
3597 ud->ddev.src_addr_widths = TI_UDMAC_BUSWIDTHS; in udma_probe()
3598 ud->ddev.dst_addr_widths = TI_UDMAC_BUSWIDTHS; in udma_probe()
3599 ud->ddev.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV); in udma_probe()
3600 ud->ddev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST; in udma_probe()
3601 ud->ddev.copy_align = DMAENGINE_ALIGN_8_BYTES; in udma_probe()
3602 ud->ddev.desc_metadata_modes = DESC_METADATA_CLIENT | in udma_probe()
3604 if (ud->match_data->enable_memcpy_support) { in udma_probe()
3605 dma_cap_set(DMA_MEMCPY, ud->ddev.cap_mask); in udma_probe()
3606 ud->ddev.device_prep_dma_memcpy = udma_prep_dma_memcpy; in udma_probe()
3607 ud->ddev.directions |= BIT(DMA_MEM_TO_MEM); in udma_probe()
3610 ud->ddev.dev = dev; in udma_probe()
3611 ud->dev = dev; in udma_probe()
3612 ud->psil_base = ud->match_data->psil_base; in udma_probe()
3614 INIT_LIST_HEAD(&ud->ddev.channels); in udma_probe()
3615 INIT_LIST_HEAD(&ud->desc_to_purge); in udma_probe()
3617 ch_count = udma_setup_resources(ud); in udma_probe()
3621 spin_lock_init(&ud->lock); in udma_probe()
3622 INIT_WORK(&ud->purge_work, udma_purge_desc_work); in udma_probe()
3624 ud->desc_align = 64; in udma_probe()
3625 if (ud->desc_align < dma_get_cache_alignment()) in udma_probe()
3626 ud->desc_align = dma_get_cache_alignment(); in udma_probe()
3628 ret = udma_setup_rx_flush(ud); in udma_probe()
3632 for (i = 0; i < ud->tchan_cnt; i++) { in udma_probe()
3633 struct udma_tchan *tchan = &ud->tchans[i]; in udma_probe()
3636 tchan->reg_rt = ud->mmrs[MMR_TCHANRT] + i * 0x1000; in udma_probe()
3639 for (i = 0; i < ud->rchan_cnt; i++) { in udma_probe()
3640 struct udma_rchan *rchan = &ud->rchans[i]; in udma_probe()
3643 rchan->reg_rt = ud->mmrs[MMR_RCHANRT] + i * 0x1000; in udma_probe()
3646 for (i = 0; i < ud->rflow_cnt; i++) { in udma_probe()
3647 struct udma_rflow *rflow = &ud->rflows[i]; in udma_probe()
3653 struct udma_chan *uc = &ud->channels[i]; in udma_probe()
3655 uc->ud = ud; in udma_probe()
3665 vchan_init(&uc->vc, &ud->ddev); in udma_probe()
3672 ret = dma_async_device_register(&ud->ddev); in udma_probe()
3678 platform_set_drvdata(pdev, ud); in udma_probe()
3680 ret = of_dma_controller_register(dev->of_node, udma_of_xlate, ud); in udma_probe()
3683 dma_async_device_unregister(&ud->ddev); in udma_probe()