Lines Matching refs:d40c
601 static struct device *chan2dev(struct d40_chan *d40c) in chan2dev() argument
603 return &d40c->chan.dev->device; in chan2dev()
625 #define chan_err(d40c, format, arg...) \ argument
626 d40_err(chan2dev(d40c), format, ## arg)
628 static int d40_pool_lli_alloc(struct d40_chan *d40c, struct d40_desc *d40d, in d40_pool_lli_alloc() argument
631 bool is_log = chan_is_logical(d40c); in d40_pool_lli_alloc()
663 d40d->lli_pool.dma_addr = dma_map_single(d40c->base->dev, in d40_pool_lli_alloc()
668 if (dma_mapping_error(d40c->base->dev, in d40_pool_lli_alloc()
680 static void d40_pool_lli_free(struct d40_chan *d40c, struct d40_desc *d40d) in d40_pool_lli_free() argument
683 dma_unmap_single(d40c->base->dev, d40d->lli_pool.dma_addr, in d40_pool_lli_free()
695 static int d40_lcla_alloc_one(struct d40_chan *d40c, in d40_lcla_alloc_one() argument
702 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); in d40_lcla_alloc_one()
709 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i; in d40_lcla_alloc_one()
711 if (!d40c->base->lcla_pool.alloc_map[idx]) { in d40_lcla_alloc_one()
712 d40c->base->lcla_pool.alloc_map[idx] = d40d; in d40_lcla_alloc_one()
719 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); in d40_lcla_alloc_one()
724 static int d40_lcla_free_all(struct d40_chan *d40c, in d40_lcla_free_all() argument
731 if (chan_is_physical(d40c)) in d40_lcla_free_all()
734 spin_lock_irqsave(&d40c->base->lcla_pool.lock, flags); in d40_lcla_free_all()
737 int idx = d40c->phy_chan->num * D40_LCLA_LINK_PER_EVENT_GRP + i; in d40_lcla_free_all()
739 if (d40c->base->lcla_pool.alloc_map[idx] == d40d) { in d40_lcla_free_all()
740 d40c->base->lcla_pool.alloc_map[idx] = NULL; in d40_lcla_free_all()
749 spin_unlock_irqrestore(&d40c->base->lcla_pool.lock, flags); in d40_lcla_free_all()
760 static struct d40_desc *d40_desc_get(struct d40_chan *d40c) in d40_desc_get() argument
764 if (!list_empty(&d40c->client)) { in d40_desc_get()
768 list_for_each_entry_safe(d, _d, &d40c->client, node) { in d40_desc_get()
779 desc = kmem_cache_zalloc(d40c->base->desc_slab, GFP_NOWAIT); in d40_desc_get()
787 static void d40_desc_free(struct d40_chan *d40c, struct d40_desc *d40d) in d40_desc_free() argument
790 d40_pool_lli_free(d40c, d40d); in d40_desc_free()
791 d40_lcla_free_all(d40c, d40d); in d40_desc_free()
792 kmem_cache_free(d40c->base->desc_slab, d40d); in d40_desc_free()
795 static void d40_desc_submit(struct d40_chan *d40c, struct d40_desc *desc) in d40_desc_submit() argument
797 list_add_tail(&desc->node, &d40c->active); in d40_desc_submit()
817 static void d40_desc_done(struct d40_chan *d40c, struct d40_desc *desc) in d40_desc_done() argument
819 list_add_tail(&desc->node, &d40c->done); in d40_desc_done()
934 static void d40_desc_load(struct d40_chan *d40c, struct d40_desc *d40d) in d40_desc_load() argument
936 if (chan_is_physical(d40c)) { in d40_desc_load()
937 d40_phy_lli_load(d40c, d40d); in d40_desc_load()
940 d40_log_lli_to_lcxa(d40c, d40d); in d40_desc_load()
943 static struct d40_desc *d40_first_active_get(struct d40_chan *d40c) in d40_first_active_get() argument
945 return list_first_entry_or_null(&d40c->active, struct d40_desc, node); in d40_first_active_get()
949 static void d40_desc_queue(struct d40_chan *d40c, struct d40_desc *desc) in d40_desc_queue() argument
953 list_add_tail(&desc->node, &d40c->pending_queue); in d40_desc_queue()
956 static struct d40_desc *d40_first_pending(struct d40_chan *d40c) in d40_first_pending() argument
958 return list_first_entry_or_null(&d40c->pending_queue, struct d40_desc, in d40_first_pending()
962 static struct d40_desc *d40_first_queued(struct d40_chan *d40c) in d40_first_queued() argument
964 return list_first_entry_or_null(&d40c->queue, struct d40_desc, node); in d40_first_queued()
967 static struct d40_desc *d40_first_done(struct d40_chan *d40c) in d40_first_done() argument
969 return list_first_entry_or_null(&d40c->done, struct d40_desc, node); in d40_first_done()
1032 static int __d40_execute_command_phy(struct d40_chan *d40c, in __d40_execute_command_phy() argument
1043 ret = __d40_execute_command_phy(d40c, D40_DMA_SUSPEND_REQ); in __d40_execute_command_phy()
1048 spin_lock_irqsave(&d40c->base->execmd_lock, flags); in __d40_execute_command_phy()
1050 if (d40c->phy_chan->num % 2 == 0) in __d40_execute_command_phy()
1051 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; in __d40_execute_command_phy()
1053 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; in __d40_execute_command_phy()
1057 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> in __d40_execute_command_phy()
1058 D40_CHAN_POS(d40c->phy_chan->num); in __d40_execute_command_phy()
1064 wmask = 0xffffffff & ~(D40_CHAN_POS_MASK(d40c->phy_chan->num)); in __d40_execute_command_phy()
1065 writel(wmask | (command << D40_CHAN_POS(d40c->phy_chan->num)), in __d40_execute_command_phy()
1072 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> in __d40_execute_command_phy()
1073 D40_CHAN_POS(d40c->phy_chan->num); in __d40_execute_command_phy()
1088 chan_err(d40c, in __d40_execute_command_phy()
1090 d40c->phy_chan->num, d40c->log_num, in __d40_execute_command_phy()
1098 spin_unlock_irqrestore(&d40c->base->execmd_lock, flags); in __d40_execute_command_phy()
1102 static void d40_term_all(struct d40_chan *d40c) in d40_term_all() argument
1108 while ((d40d = d40_first_done(d40c))) { in d40_term_all()
1110 d40_desc_free(d40c, d40d); in d40_term_all()
1114 while ((d40d = d40_first_active_get(d40c))) { in d40_term_all()
1116 d40_desc_free(d40c, d40d); in d40_term_all()
1120 while ((d40d = d40_first_queued(d40c))) { in d40_term_all()
1122 d40_desc_free(d40c, d40d); in d40_term_all()
1126 while ((d40d = d40_first_pending(d40c))) { in d40_term_all()
1128 d40_desc_free(d40c, d40d); in d40_term_all()
1132 if (!list_empty(&d40c->client)) in d40_term_all()
1133 list_for_each_entry_safe(d40d, _d, &d40c->client, node) { in d40_term_all()
1135 d40_desc_free(d40c, d40d); in d40_term_all()
1139 if (!list_empty(&d40c->prepare_queue)) in d40_term_all()
1141 &d40c->prepare_queue, node) { in d40_term_all()
1143 d40_desc_free(d40c, d40d); in d40_term_all()
1146 d40c->pending_tx = 0; in d40_term_all()
1149 static void __d40_config_set_event(struct d40_chan *d40c, in __d40_config_set_event() argument
1153 void __iomem *addr = chan_base(d40c) + reg; in __d40_config_set_event()
1193 chan_err(d40c, in __d40_config_set_event()
1195 "status %x\n", d40c->phy_chan->num, in __d40_config_set_event()
1196 d40c->log_num, status); in __d40_config_set_event()
1217 dev_dbg(chan2dev(d40c), in __d40_config_set_event()
1232 static void d40_config_set_event(struct d40_chan *d40c, in d40_config_set_event() argument
1235 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type); in d40_config_set_event()
1238 if ((d40c->dma_cfg.dir == DMA_DEV_TO_MEM) || in d40_config_set_event()
1239 (d40c->dma_cfg.dir == DMA_DEV_TO_DEV)) in d40_config_set_event()
1240 __d40_config_set_event(d40c, event_type, event, in d40_config_set_event()
1243 if (d40c->dma_cfg.dir != DMA_DEV_TO_MEM) in d40_config_set_event()
1244 __d40_config_set_event(d40c, event_type, event, in d40_config_set_event()
1248 static u32 d40_chan_has_events(struct d40_chan *d40c) in d40_chan_has_events() argument
1250 void __iomem *chanbase = chan_base(d40c); in d40_chan_has_events()
1260 __d40_execute_command_log(struct d40_chan *d40c, enum d40_command command) in __d40_execute_command_log() argument
1267 if (d40c->phy_chan->num % 2 == 0) in __d40_execute_command_log()
1268 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; in __d40_execute_command_log()
1270 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; in __d40_execute_command_log()
1273 spin_lock_irqsave(&d40c->phy_chan->lock, flags); in __d40_execute_command_log()
1280 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> in __d40_execute_command_log()
1281 D40_CHAN_POS(d40c->phy_chan->num); in __d40_execute_command_log()
1284 d40_config_set_event(d40c, D40_SUSPEND_REQ_EVENTLINE); in __d40_execute_command_log()
1286 d40_config_set_event(d40c, D40_DEACTIVATE_EVENTLINE); in __d40_execute_command_log()
1288 if (!d40_chan_has_events(d40c) && (command == D40_DMA_STOP)) in __d40_execute_command_log()
1289 ret = __d40_execute_command_phy(d40c, command); in __d40_execute_command_log()
1295 d40_config_set_event(d40c, D40_ACTIVATE_EVENTLINE); in __d40_execute_command_log()
1296 ret = __d40_execute_command_phy(d40c, command); in __d40_execute_command_log()
1304 spin_unlock_irqrestore(&d40c->phy_chan->lock, flags); in __d40_execute_command_log()
1308 static int d40_channel_execute_command(struct d40_chan *d40c, in d40_channel_execute_command() argument
1311 if (chan_is_logical(d40c)) in d40_channel_execute_command()
1312 return __d40_execute_command_log(d40c, command); in d40_channel_execute_command()
1314 return __d40_execute_command_phy(d40c, command); in d40_channel_execute_command()
1317 static u32 d40_get_prmo(struct d40_chan *d40c) in d40_get_prmo() argument
1336 if (chan_is_physical(d40c)) in d40_get_prmo()
1337 return phy_map[d40c->dma_cfg.mode_opt]; in d40_get_prmo()
1339 return log_map[d40c->dma_cfg.mode_opt]; in d40_get_prmo()
1342 static void d40_config_write(struct d40_chan *d40c) in d40_config_write() argument
1348 addr_base = (d40c->phy_chan->num % 2) * 4; in d40_config_write()
1350 var = ((u32)(chan_is_logical(d40c)) + 1) << in d40_config_write()
1351 D40_CHAN_POS(d40c->phy_chan->num); in d40_config_write()
1352 writel(var, d40c->base->virtbase + D40_DREG_PRMSE + addr_base); in d40_config_write()
1355 var = d40_get_prmo(d40c) << D40_CHAN_POS(d40c->phy_chan->num); in d40_config_write()
1357 writel(var, d40c->base->virtbase + D40_DREG_PRMOE + addr_base); in d40_config_write()
1359 if (chan_is_logical(d40c)) { in d40_config_write()
1360 int lidx = (d40c->phy_chan->num << D40_SREG_ELEM_LOG_LIDX_POS) in d40_config_write()
1362 void __iomem *chanbase = chan_base(d40c); in d40_config_write()
1365 writel(d40c->src_def_cfg, chanbase + D40_CHAN_REG_SSCFG); in d40_config_write()
1366 writel(d40c->dst_def_cfg, chanbase + D40_CHAN_REG_SDCFG); in d40_config_write()
1378 static u32 d40_residue(struct d40_chan *d40c) in d40_residue() argument
1382 if (chan_is_logical(d40c)) in d40_residue()
1383 num_elt = (readl(&d40c->lcpa->lcsp2) & D40_MEM_LCSP2_ECNT_MASK) in d40_residue()
1386 u32 val = readl(chan_base(d40c) + D40_CHAN_REG_SDELT); in d40_residue()
1391 return num_elt * d40c->dma_cfg.dst_info.data_width; in d40_residue()
1394 static bool d40_tx_is_linked(struct d40_chan *d40c) in d40_tx_is_linked() argument
1398 if (chan_is_logical(d40c)) in d40_tx_is_linked()
1399 is_link = readl(&d40c->lcpa->lcsp3) & D40_MEM_LCSP3_DLOS_MASK; in d40_tx_is_linked()
1401 is_link = readl(chan_base(d40c) + D40_CHAN_REG_SDLNK) in d40_tx_is_linked()
1409 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_pause() local
1413 if (d40c->phy_chan == NULL) { in d40_pause()
1414 chan_err(d40c, "Channel is not allocated!\n"); in d40_pause()
1418 if (!d40c->busy) in d40_pause()
1421 spin_lock_irqsave(&d40c->lock, flags); in d40_pause()
1422 pm_runtime_get_sync(d40c->base->dev); in d40_pause()
1424 res = d40_channel_execute_command(d40c, D40_DMA_SUSPEND_REQ); in d40_pause()
1426 pm_runtime_mark_last_busy(d40c->base->dev); in d40_pause()
1427 pm_runtime_put_autosuspend(d40c->base->dev); in d40_pause()
1428 spin_unlock_irqrestore(&d40c->lock, flags); in d40_pause()
1434 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_resume() local
1438 if (d40c->phy_chan == NULL) { in d40_resume()
1439 chan_err(d40c, "Channel is not allocated!\n"); in d40_resume()
1443 if (!d40c->busy) in d40_resume()
1446 spin_lock_irqsave(&d40c->lock, flags); in d40_resume()
1447 pm_runtime_get_sync(d40c->base->dev); in d40_resume()
1450 if (d40_residue(d40c) || d40_tx_is_linked(d40c)) in d40_resume()
1451 res = d40_channel_execute_command(d40c, D40_DMA_RUN); in d40_resume()
1453 pm_runtime_mark_last_busy(d40c->base->dev); in d40_resume()
1454 pm_runtime_put_autosuspend(d40c->base->dev); in d40_resume()
1455 spin_unlock_irqrestore(&d40c->lock, flags); in d40_resume()
1461 struct d40_chan *d40c = container_of(tx->chan, in d40_tx_submit() local
1468 spin_lock_irqsave(&d40c->lock, flags); in d40_tx_submit()
1470 d40_desc_queue(d40c, d40d); in d40_tx_submit()
1471 spin_unlock_irqrestore(&d40c->lock, flags); in d40_tx_submit()
1476 static int d40_start(struct d40_chan *d40c) in d40_start() argument
1478 return d40_channel_execute_command(d40c, D40_DMA_RUN); in d40_start()
1481 static struct d40_desc *d40_queue_start(struct d40_chan *d40c) in d40_queue_start() argument
1487 d40d = d40_first_queued(d40c); in d40_queue_start()
1490 if (!d40c->busy) { in d40_queue_start()
1491 d40c->busy = true; in d40_queue_start()
1492 pm_runtime_get_sync(d40c->base->dev); in d40_queue_start()
1499 d40_desc_submit(d40c, d40d); in d40_queue_start()
1502 d40_desc_load(d40c, d40d); in d40_queue_start()
1505 err = d40_start(d40c); in d40_queue_start()
1515 static void dma_tc_handle(struct d40_chan *d40c) in dma_tc_handle() argument
1520 d40d = d40_first_active_get(d40c); in dma_tc_handle()
1533 && !d40_tx_is_linked(d40c) in dma_tc_handle()
1534 && !d40_residue(d40c)) { in dma_tc_handle()
1535 d40_lcla_free_all(d40c, d40d); in dma_tc_handle()
1536 d40_desc_load(d40c, d40d); in dma_tc_handle()
1537 (void) d40_start(d40c); in dma_tc_handle()
1543 d40_lcla_free_all(d40c, d40d); in dma_tc_handle()
1546 d40_desc_load(d40c, d40d); in dma_tc_handle()
1548 (void) d40_start(d40c); in dma_tc_handle()
1552 if (d40_queue_start(d40c) == NULL) { in dma_tc_handle()
1553 d40c->busy = false; in dma_tc_handle()
1555 pm_runtime_mark_last_busy(d40c->base->dev); in dma_tc_handle()
1556 pm_runtime_put_autosuspend(d40c->base->dev); in dma_tc_handle()
1560 d40_desc_done(d40c, d40d); in dma_tc_handle()
1563 d40c->pending_tx++; in dma_tc_handle()
1564 tasklet_schedule(&d40c->tasklet); in dma_tc_handle()
1570 struct d40_chan *d40c = (struct d40_chan *) data; in dma_tasklet() local
1576 spin_lock_irqsave(&d40c->lock, flags); in dma_tasklet()
1579 d40d = d40_first_done(d40c); in dma_tasklet()
1582 d40d = d40_first_active_get(d40c); in dma_tasklet()
1594 if (d40c->pending_tx == 0) { in dma_tasklet()
1595 spin_unlock_irqrestore(&d40c->lock, flags); in dma_tasklet()
1606 d40_desc_free(d40c, d40d); in dma_tasklet()
1609 d40_lcla_free_all(d40c, d40d); in dma_tasklet()
1610 list_add_tail(&d40d->node, &d40c->client); in dma_tasklet()
1615 d40c->pending_tx--; in dma_tasklet()
1617 if (d40c->pending_tx) in dma_tasklet()
1618 tasklet_schedule(&d40c->tasklet); in dma_tasklet()
1620 spin_unlock_irqrestore(&d40c->lock, flags); in dma_tasklet()
1628 if (d40c->pending_tx > 0) in dma_tasklet()
1629 d40c->pending_tx--; in dma_tasklet()
1630 spin_unlock_irqrestore(&d40c->lock, flags); in dma_tasklet()
1639 struct d40_chan *d40c; in d40_handle_interrupt() local
1665 d40c = base->lookup_phy_chans[idx]; in d40_handle_interrupt()
1667 d40c = base->lookup_log_chans[il[row].offset + idx]; in d40_handle_interrupt()
1669 if (!d40c) { in d40_handle_interrupt()
1680 spin_lock(&d40c->lock); in d40_handle_interrupt()
1683 dma_tc_handle(d40c); in d40_handle_interrupt()
1688 spin_unlock(&d40c->lock); in d40_handle_interrupt()
1696 static int d40_validate_conf(struct d40_chan *d40c, in d40_validate_conf() argument
1703 chan_err(d40c, "Invalid direction.\n"); in d40_validate_conf()
1707 if ((is_log && conf->dev_type > d40c->base->num_log_chans) || in d40_validate_conf()
1708 (!is_log && conf->dev_type > d40c->base->num_phy_chans) || in d40_validate_conf()
1710 chan_err(d40c, "Invalid device type (%d)\n", conf->dev_type); in d40_validate_conf()
1719 chan_err(d40c, "periph to periph not supported\n"); in d40_validate_conf()
1732 chan_err(d40c, "src (burst x width) != dst (burst x width)\n"); in d40_validate_conf()
1826 static int d40_allocate_channel(struct d40_chan *d40c, bool *first_phy_user) in d40_allocate_channel() argument
1828 int dev_type = d40c->dma_cfg.dev_type; in d40_allocate_channel()
1837 bool is_log = d40c->dma_cfg.mode == STEDMA40_MODE_LOGICAL; in d40_allocate_channel()
1839 phys = d40c->base->phy_res; in d40_allocate_channel()
1840 num_phy_chans = d40c->base->num_phy_chans; in d40_allocate_channel()
1842 if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) { in d40_allocate_channel()
1845 } else if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV || in d40_allocate_channel()
1846 d40c->dma_cfg.dir == DMA_MEM_TO_MEM) { in d40_allocate_channel()
1857 if (d40c->dma_cfg.dir == DMA_MEM_TO_MEM) { in d40_allocate_channel()
1859 if (d40c->dma_cfg.use_fixed_channel) { in d40_allocate_channel()
1860 i = d40c->dma_cfg.phy_channel; in d40_allocate_channel()
1874 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { in d40_allocate_channel()
1887 d40c->phy_chan = &phys[i]; in d40_allocate_channel()
1888 d40c->log_num = D40_PHY_CHAN; in d40_allocate_channel()
1895 for (j = 0; j < d40c->base->num_phy_chans; j += 8) { in d40_allocate_channel()
1898 if (d40c->dma_cfg.use_fixed_channel) { in d40_allocate_channel()
1899 i = d40c->dma_cfg.phy_channel; in d40_allocate_channel()
1902 dev_err(chan2dev(d40c), in d40_allocate_channel()
1911 dev_err(chan2dev(d40c), in d40_allocate_channel()
1940 d40c->phy_chan = &phys[i]; in d40_allocate_channel()
1941 d40c->log_num = log_num; in d40_allocate_channel()
1945 d40c->base->lookup_log_chans[d40c->log_num] = d40c; in d40_allocate_channel()
1947 d40c->base->lookup_phy_chans[d40c->phy_chan->num] = d40c; in d40_allocate_channel()
1953 static int d40_config_memcpy(struct d40_chan *d40c) in d40_config_memcpy() argument
1955 dma_cap_mask_t cap = d40c->chan.device->cap_mask; in d40_config_memcpy()
1958 d40c->dma_cfg = dma40_memcpy_conf_log; in d40_config_memcpy()
1959 d40c->dma_cfg.dev_type = dma40_memcpy_channels[d40c->chan.chan_id]; in d40_config_memcpy()
1961 d40_log_cfg(&d40c->dma_cfg, in d40_config_memcpy()
1962 &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); in d40_config_memcpy()
1966 d40c->dma_cfg = dma40_memcpy_conf_phy; in d40_config_memcpy()
1969 d40c->dst_def_cfg |= BIT(D40_SREG_CFG_TIM_POS); in d40_config_memcpy()
1972 d40c->src_def_cfg |= BIT(D40_SREG_CFG_EIM_POS); in d40_config_memcpy()
1973 d40c->dst_def_cfg |= BIT(D40_SREG_CFG_EIM_POS); in d40_config_memcpy()
1976 chan_err(d40c, "No memcpy\n"); in d40_config_memcpy()
1983 static int d40_free_dma(struct d40_chan *d40c) in d40_free_dma() argument
1987 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type); in d40_free_dma()
1988 struct d40_phy_res *phy = d40c->phy_chan; in d40_free_dma()
1992 d40_term_all(d40c); in d40_free_dma()
1995 chan_err(d40c, "phy == null\n"); in d40_free_dma()
2001 chan_err(d40c, "channel already free\n"); in d40_free_dma()
2005 if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV || in d40_free_dma()
2006 d40c->dma_cfg.dir == DMA_MEM_TO_MEM) in d40_free_dma()
2008 else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) in d40_free_dma()
2011 chan_err(d40c, "Unknown direction\n"); in d40_free_dma()
2015 pm_runtime_get_sync(d40c->base->dev); in d40_free_dma()
2016 res = d40_channel_execute_command(d40c, D40_DMA_STOP); in d40_free_dma()
2018 chan_err(d40c, "stop failed\n"); in d40_free_dma()
2022 d40_alloc_mask_free(phy, is_src, chan_is_logical(d40c) ? event : 0); in d40_free_dma()
2024 if (chan_is_logical(d40c)) in d40_free_dma()
2025 d40c->base->lookup_log_chans[d40c->log_num] = NULL; in d40_free_dma()
2027 d40c->base->lookup_phy_chans[phy->num] = NULL; in d40_free_dma()
2029 if (d40c->busy) { in d40_free_dma()
2030 pm_runtime_mark_last_busy(d40c->base->dev); in d40_free_dma()
2031 pm_runtime_put_autosuspend(d40c->base->dev); in d40_free_dma()
2034 d40c->busy = false; in d40_free_dma()
2035 d40c->phy_chan = NULL; in d40_free_dma()
2036 d40c->configured = false; in d40_free_dma()
2038 pm_runtime_mark_last_busy(d40c->base->dev); in d40_free_dma()
2039 pm_runtime_put_autosuspend(d40c->base->dev); in d40_free_dma()
2043 static bool d40_is_paused(struct d40_chan *d40c) in d40_is_paused() argument
2045 void __iomem *chanbase = chan_base(d40c); in d40_is_paused()
2050 u32 event = D40_TYPE_TO_EVENT(d40c->dma_cfg.dev_type); in d40_is_paused()
2052 spin_lock_irqsave(&d40c->lock, flags); in d40_is_paused()
2054 if (chan_is_physical(d40c)) { in d40_is_paused()
2055 if (d40c->phy_chan->num % 2 == 0) in d40_is_paused()
2056 active_reg = d40c->base->virtbase + D40_DREG_ACTIVE; in d40_is_paused()
2058 active_reg = d40c->base->virtbase + D40_DREG_ACTIVO; in d40_is_paused()
2061 D40_CHAN_POS_MASK(d40c->phy_chan->num)) >> in d40_is_paused()
2062 D40_CHAN_POS(d40c->phy_chan->num); in d40_is_paused()
2068 if (d40c->dma_cfg.dir == DMA_MEM_TO_DEV || in d40_is_paused()
2069 d40c->dma_cfg.dir == DMA_MEM_TO_MEM) { in d40_is_paused()
2071 } else if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) { in d40_is_paused()
2074 chan_err(d40c, "Unknown direction\n"); in d40_is_paused()
2084 spin_unlock_irqrestore(&d40c->lock, flags); in d40_is_paused()
2091 struct d40_chan *d40c = in stedma40_residue() local
2096 spin_lock_irqsave(&d40c->lock, flags); in stedma40_residue()
2097 bytes_left = d40_residue(d40c); in stedma40_residue()
2098 spin_unlock_irqrestore(&d40c->lock, flags); in stedma40_residue()
2267 struct d40_chan *d40c = in stedma40_filter() local
2272 err = d40_validate_conf(d40c, info); in stedma40_filter()
2274 d40c->dma_cfg = *info; in stedma40_filter()
2276 err = d40_config_memcpy(d40c); in stedma40_filter()
2279 d40c->configured = true; in stedma40_filter()
2285 static void __d40_set_prio_rt(struct d40_chan *d40c, int dev_type, bool src) in __d40_set_prio_rt() argument
2287 bool realtime = d40c->dma_cfg.realtime; in __d40_set_prio_rt()
2288 bool highprio = d40c->dma_cfg.high_priority; in __d40_set_prio_rt()
2294 struct d40_gen_dmac *dmac = &d40c->base->gen_dmac; in __d40_set_prio_rt()
2305 if (!src && chan_is_logical(d40c)) in __d40_set_prio_rt()
2314 writel(bit, d40c->base->virtbase + prioreg + group * 4); in __d40_set_prio_rt()
2315 writel(bit, d40c->base->virtbase + rtreg + group * 4); in __d40_set_prio_rt()
2318 static void d40_set_prio_realtime(struct d40_chan *d40c) in d40_set_prio_realtime() argument
2320 if (d40c->base->rev < 3) in d40_set_prio_realtime()
2323 if ((d40c->dma_cfg.dir == DMA_DEV_TO_MEM) || in d40_set_prio_realtime()
2324 (d40c->dma_cfg.dir == DMA_DEV_TO_DEV)) in d40_set_prio_realtime()
2325 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, true); in d40_set_prio_realtime()
2327 if ((d40c->dma_cfg.dir == DMA_MEM_TO_DEV) || in d40_set_prio_realtime()
2328 (d40c->dma_cfg.dir == DMA_DEV_TO_DEV)) in d40_set_prio_realtime()
2329 __d40_set_prio_rt(d40c, d40c->dma_cfg.dev_type, false); in d40_set_prio_realtime()
2385 struct d40_chan *d40c = in d40_alloc_chan_resources() local
2388 spin_lock_irqsave(&d40c->lock, flags); in d40_alloc_chan_resources()
2393 if (!d40c->configured) { in d40_alloc_chan_resources()
2394 err = d40_config_memcpy(d40c); in d40_alloc_chan_resources()
2396 chan_err(d40c, "Failed to configure memcpy channel\n"); in d40_alloc_chan_resources()
2401 err = d40_allocate_channel(d40c, &is_free_phy); in d40_alloc_chan_resources()
2403 chan_err(d40c, "Failed to allocate channel\n"); in d40_alloc_chan_resources()
2404 d40c->configured = false; in d40_alloc_chan_resources()
2408 pm_runtime_get_sync(d40c->base->dev); in d40_alloc_chan_resources()
2410 d40_set_prio_realtime(d40c); in d40_alloc_chan_resources()
2412 if (chan_is_logical(d40c)) { in d40_alloc_chan_resources()
2413 if (d40c->dma_cfg.dir == DMA_DEV_TO_MEM) in d40_alloc_chan_resources()
2414 d40c->lcpa = d40c->base->lcpa_base + in d40_alloc_chan_resources()
2415 d40c->dma_cfg.dev_type * D40_LCPA_CHAN_SIZE; in d40_alloc_chan_resources()
2417 d40c->lcpa = d40c->base->lcpa_base + in d40_alloc_chan_resources()
2418 d40c->dma_cfg.dev_type * in d40_alloc_chan_resources()
2422 d40c->src_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS); in d40_alloc_chan_resources()
2423 d40c->dst_def_cfg |= BIT(D40_SREG_CFG_LOG_GIM_POS); in d40_alloc_chan_resources()
2426 dev_dbg(chan2dev(d40c), "allocated %s channel (phy %d%s)\n", in d40_alloc_chan_resources()
2427 chan_is_logical(d40c) ? "logical" : "physical", in d40_alloc_chan_resources()
2428 d40c->phy_chan->num, in d40_alloc_chan_resources()
2429 d40c->dma_cfg.use_fixed_channel ? ", fixed" : ""); in d40_alloc_chan_resources()
2438 d40_config_write(d40c); in d40_alloc_chan_resources()
2440 pm_runtime_mark_last_busy(d40c->base->dev); in d40_alloc_chan_resources()
2441 pm_runtime_put_autosuspend(d40c->base->dev); in d40_alloc_chan_resources()
2442 spin_unlock_irqrestore(&d40c->lock, flags); in d40_alloc_chan_resources()
2448 struct d40_chan *d40c = in d40_free_chan_resources() local
2453 if (d40c->phy_chan == NULL) { in d40_free_chan_resources()
2454 chan_err(d40c, "Cannot free unallocated channel\n"); in d40_free_chan_resources()
2458 spin_lock_irqsave(&d40c->lock, flags); in d40_free_chan_resources()
2460 err = d40_free_dma(d40c); in d40_free_chan_resources()
2463 chan_err(d40c, "Failed to free channel\n"); in d40_free_chan_resources()
2464 spin_unlock_irqrestore(&d40c->lock, flags); in d40_free_chan_resources()
2534 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_tx_status() local
2537 if (d40c->phy_chan == NULL) { in d40_tx_status()
2538 chan_err(d40c, "Cannot read status of unallocated channel\n"); in d40_tx_status()
2546 if (d40_is_paused(d40c)) in d40_tx_status()
2554 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_issue_pending() local
2557 if (d40c->phy_chan == NULL) { in d40_issue_pending()
2558 chan_err(d40c, "Channel is not allocated!\n"); in d40_issue_pending()
2562 spin_lock_irqsave(&d40c->lock, flags); in d40_issue_pending()
2564 list_splice_tail_init(&d40c->pending_queue, &d40c->queue); in d40_issue_pending()
2567 if (!d40c->busy) in d40_issue_pending()
2568 (void) d40_queue_start(d40c); in d40_issue_pending()
2570 spin_unlock_irqrestore(&d40c->lock, flags); in d40_issue_pending()
2576 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_terminate_all() local
2579 if (d40c->phy_chan == NULL) { in d40_terminate_all()
2580 chan_err(d40c, "Channel is not allocated!\n"); in d40_terminate_all()
2584 spin_lock_irqsave(&d40c->lock, flags); in d40_terminate_all()
2586 pm_runtime_get_sync(d40c->base->dev); in d40_terminate_all()
2587 ret = d40_channel_execute_command(d40c, D40_DMA_STOP); in d40_terminate_all()
2589 chan_err(d40c, "Failed to stop channel\n"); in d40_terminate_all()
2591 d40_term_all(d40c); in d40_terminate_all()
2592 pm_runtime_mark_last_busy(d40c->base->dev); in d40_terminate_all()
2593 pm_runtime_put_autosuspend(d40c->base->dev); in d40_terminate_all()
2594 if (d40c->busy) { in d40_terminate_all()
2595 pm_runtime_mark_last_busy(d40c->base->dev); in d40_terminate_all()
2596 pm_runtime_put_autosuspend(d40c->base->dev); in d40_terminate_all()
2598 d40c->busy = false; in d40_terminate_all()
2600 spin_unlock_irqrestore(&d40c->lock, flags); in d40_terminate_all()
2605 dma40_config_to_halfchannel(struct d40_chan *d40c, in dma40_config_to_halfchannel() argument
2611 if (chan_is_logical(d40c)) { in dma40_config_to_halfchannel()
2641 struct d40_chan *d40c = container_of(chan, struct d40_chan, chan); in d40_set_runtime_config() local
2642 struct stedma40_chan_cfg *cfg = &d40c->dma_cfg; in d40_set_runtime_config()
2648 if (d40c->phy_chan == NULL) { in d40_set_runtime_config()
2649 chan_err(d40c, "Channel is not allocated!\n"); in d40_set_runtime_config()
2662 dev_dbg(d40c->base->dev, in d40_set_runtime_config()
2678 dev_dbg(d40c->base->dev, in d40_set_runtime_config()
2690 dev_err(d40c->base->dev, in d40_set_runtime_config()
2697 dev_err(d40c->base->dev, "no address supplied\n"); in d40_set_runtime_config()
2702 dev_err(d40c->base->dev, in d40_set_runtime_config()
2731 ret = dma40_config_to_halfchannel(d40c, &cfg->src_info, in d40_set_runtime_config()
2736 ret = dma40_config_to_halfchannel(d40c, &cfg->dst_info, in d40_set_runtime_config()
2742 if (chan_is_logical(d40c)) in d40_set_runtime_config()
2743 d40_log_cfg(cfg, &d40c->log_def.lcsp1, &d40c->log_def.lcsp3); in d40_set_runtime_config()
2745 d40_phy_cfg(cfg, &d40c->src_def_cfg, &d40c->dst_def_cfg); in d40_set_runtime_config()
2748 d40c->runtime_addr = config_addr; in d40_set_runtime_config()
2749 d40c->runtime_direction = config->direction; in d40_set_runtime_config()
2750 dev_dbg(d40c->base->dev, in d40_set_runtime_config()
2768 struct d40_chan *d40c; in d40_chan_init() local
2773 d40c = &chans[i]; in d40_chan_init()
2774 d40c->base = base; in d40_chan_init()
2775 d40c->chan.device = dma; in d40_chan_init()
2777 spin_lock_init(&d40c->lock); in d40_chan_init()
2779 d40c->log_num = D40_PHY_CHAN; in d40_chan_init()
2781 INIT_LIST_HEAD(&d40c->done); in d40_chan_init()
2782 INIT_LIST_HEAD(&d40c->active); in d40_chan_init()
2783 INIT_LIST_HEAD(&d40c->queue); in d40_chan_init()
2784 INIT_LIST_HEAD(&d40c->pending_queue); in d40_chan_init()
2785 INIT_LIST_HEAD(&d40c->client); in d40_chan_init()
2786 INIT_LIST_HEAD(&d40c->prepare_queue); in d40_chan_init()
2788 tasklet_init(&d40c->tasklet, dma_tasklet, in d40_chan_init()
2789 (unsigned long) d40c); in d40_chan_init()
2791 list_add_tail(&d40c->chan.device_node, in d40_chan_init()