326 	struct sdma_channel	*sdmac;  member
623 static int sdma_config_ownership(struct sdma_channel *sdmac,  in sdma_config_ownership()  argument
626 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_config_ownership()
627 	int channel = sdmac->channel;  in sdma_config_ownership()
722 static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_enable()  argument
724 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_enable()
725 	int channel = sdmac->channel;  in sdma_event_enable()
734 static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_disable()  argument
736 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_disable()
737 	int channel = sdmac->channel;  in sdma_event_disable()
751 static void sdma_start_desc(struct sdma_channel *sdmac)  in sdma_start_desc()  argument
753 	struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);  in sdma_start_desc()
755 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_start_desc()
756 	int channel = sdmac->channel;  in sdma_start_desc()
759 		sdmac->desc = NULL;  in sdma_start_desc()
762 	sdmac->desc = desc = to_sdma_desc(&vd->tx);  in sdma_start_desc()
767 	if (!(sdmac->flags & IMX_DMA_SG_LOOP))  in sdma_start_desc()
772 	sdma_enable_channel(sdma, sdmac->channel);  in sdma_start_desc()
775 static void sdma_update_channel_loop(struct sdma_channel *sdmac)  in sdma_update_channel_loop()  argument
779 	enum dma_status	old_status = sdmac->status;  in sdma_update_channel_loop()
785 	while (sdmac->desc) {  in sdma_update_channel_loop()
786 		struct sdma_desc *desc = sdmac->desc;  in sdma_update_channel_loop()
795 			sdmac->status = DMA_ERROR;  in sdma_update_channel_loop()
816 		spin_unlock(&sdmac->vc.lock);  in sdma_update_channel_loop()
818 		spin_lock(&sdmac->vc.lock);  in sdma_update_channel_loop()
821 			sdmac->status = old_status;  in sdma_update_channel_loop()
827 	struct sdma_channel *sdmac = (struct sdma_channel *) data;  in mxc_sdma_handle_channel_normal()  local
831 	sdmac->desc->chn_real_count = 0;  in mxc_sdma_handle_channel_normal()
836 	for (i = 0; i < sdmac->desc->num_bd; i++) {  in mxc_sdma_handle_channel_normal()
837 		bd = &sdmac->desc->bd[i];  in mxc_sdma_handle_channel_normal()
841 		 sdmac->desc->chn_real_count += bd->mode.count;  in mxc_sdma_handle_channel_normal()
845 		sdmac->status = DMA_ERROR;  in mxc_sdma_handle_channel_normal()
847 		sdmac->status = DMA_COMPLETE;  in mxc_sdma_handle_channel_normal()
862 		struct sdma_channel *sdmac = &sdma->channel[channel];  in sdma_int_handler()  local
865 		spin_lock(&sdmac->vc.lock);  in sdma_int_handler()
866 		desc = sdmac->desc;  in sdma_int_handler()
868 			if (sdmac->flags & IMX_DMA_SG_LOOP) {  in sdma_int_handler()
869 				sdma_update_channel_loop(sdmac);  in sdma_int_handler()
871 				mxc_sdma_handle_channel_normal(sdmac);  in sdma_int_handler()
873 				sdma_start_desc(sdmac);  in sdma_int_handler()
877 		spin_unlock(&sdmac->vc.lock);  in sdma_int_handler()
887 static void sdma_get_pc(struct sdma_channel *sdmac,  in sdma_get_pc()  argument
890 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_get_pc()
898 	sdmac->pc_from_device = 0;  in sdma_get_pc()
899 	sdmac->pc_to_device = 0;  in sdma_get_pc()
900 	sdmac->device_to_device = 0;  in sdma_get_pc()
901 	sdmac->pc_to_pc = 0;  in sdma_get_pc()
975 	sdmac->pc_from_device = per_2_emi;  in sdma_get_pc()
976 	sdmac->pc_to_device = emi_2_per;  in sdma_get_pc()
977 	sdmac->device_to_device = per_2_per;  in sdma_get_pc()
978 	sdmac->pc_to_pc = emi_2_emi;  in sdma_get_pc()
981 static int sdma_load_context(struct sdma_channel *sdmac)  in sdma_load_context()  argument
983 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_load_context()
984 	int channel = sdmac->channel;  in sdma_load_context()
991 	if (sdmac->context_loaded)  in sdma_load_context()
994 	if (sdmac->direction == DMA_DEV_TO_MEM)  in sdma_load_context()
995 		load_address = sdmac->pc_from_device;  in sdma_load_context()
996 	else if (sdmac->direction == DMA_DEV_TO_DEV)  in sdma_load_context()
997 		load_address = sdmac->device_to_device;  in sdma_load_context()
998 	else if (sdmac->direction == DMA_MEM_TO_MEM)  in sdma_load_context()
999 		load_address = sdmac->pc_to_pc;  in sdma_load_context()
1001 		load_address = sdmac->pc_to_device;  in sdma_load_context()
1007 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);  in sdma_load_context()
1008 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);  in sdma_load_context()
1009 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);  in sdma_load_context()
1010 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);  in sdma_load_context()
1011 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);  in sdma_load_context()
1021 	context->gReg[0] = sdmac->event_mask[1];  in sdma_load_context()
1022 	context->gReg[1] = sdmac->event_mask[0];  in sdma_load_context()
1023 	context->gReg[2] = sdmac->per_addr;  in sdma_load_context()
1024 	context->gReg[6] = sdmac->shp_addr;  in sdma_load_context()
1025 	context->gReg[7] = sdmac->watermark_level;  in sdma_load_context()
1036 	sdmac->context_loaded = true;  in sdma_load_context()
1048 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel()  local
1049 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_disable_channel()
1050 	int channel = sdmac->channel;  in sdma_disable_channel()
1053 	sdmac->status = DMA_ERROR;  in sdma_disable_channel()
1059 	struct sdma_channel *sdmac = container_of(work, struct sdma_channel,  in sdma_channel_terminate_work()  local
1072 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_channel_terminate_work()
1073 	vchan_get_all_descriptors(&sdmac->vc, &head);  in sdma_channel_terminate_work()
1074 	sdmac->desc = NULL;  in sdma_channel_terminate_work()
1075 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_channel_terminate_work()
1076 	vchan_dma_desc_free_list(&sdmac->vc, &head);  in sdma_channel_terminate_work()
1077 	sdmac->context_loaded = false;  in sdma_channel_terminate_work()
1082 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel_async()  local
1086 	if (sdmac->desc)  in sdma_disable_channel_async()
1087 		schedule_work(&sdmac->terminate_worker);  in sdma_disable_channel_async()
1094 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_channel_synchronize()  local
1096 	vchan_synchronize(&sdmac->vc);  in sdma_channel_synchronize()
1098 	flush_work(&sdmac->terminate_worker);  in sdma_channel_synchronize()
1101 static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)  in sdma_set_watermarklevel_for_p2p()  argument
1103 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_watermarklevel_for_p2p()
1105 	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;  in sdma_set_watermarklevel_for_p2p()
1106 	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;  in sdma_set_watermarklevel_for_p2p()
1108 	set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
1109 	set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);  in sdma_set_watermarklevel_for_p2p()
1111 	if (sdmac->event_id0 > 31)  in sdma_set_watermarklevel_for_p2p()
1112 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;  in sdma_set_watermarklevel_for_p2p()
1114 	if (sdmac->event_id1 > 31)  in sdma_set_watermarklevel_for_p2p()
1115 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;  in sdma_set_watermarklevel_for_p2p()
1123 		sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |  in sdma_set_watermarklevel_for_p2p()
1125 		sdmac->watermark_level |= hwml;  in sdma_set_watermarklevel_for_p2p()
1126 		sdmac->watermark_level |= lwml << 16;  in sdma_set_watermarklevel_for_p2p()
1127 		swap(sdmac->event_mask[0], sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
1130 	if (sdmac->per_address2 >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
1131 			sdmac->per_address2 <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
1132 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;  in sdma_set_watermarklevel_for_p2p()
1134 	if (sdmac->per_address >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
1135 			sdmac->per_address <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
1136 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;  in sdma_set_watermarklevel_for_p2p()
1138 	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;  in sdma_set_watermarklevel_for_p2p()
1143 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config_channel()  local
1148 	sdmac->event_mask[0] = 0;  in sdma_config_channel()
1149 	sdmac->event_mask[1] = 0;  in sdma_config_channel()
1150 	sdmac->shp_addr = 0;  in sdma_config_channel()
1151 	sdmac->per_addr = 0;  in sdma_config_channel()
1153 	switch (sdmac->peripheral_type) {  in sdma_config_channel()
1155 		sdma_config_ownership(sdmac, false, true, true);  in sdma_config_channel()
1158 		sdma_config_ownership(sdmac, false, true, false);  in sdma_config_channel()
1161 		sdma_config_ownership(sdmac, true, true, false);  in sdma_config_channel()
1165 	sdma_get_pc(sdmac, sdmac->peripheral_type);  in sdma_config_channel()
1167 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&  in sdma_config_channel()
1168 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {  in sdma_config_channel()
1170 		if (sdmac->event_id1) {  in sdma_config_channel()
1171 			if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||  in sdma_config_channel()
1172 			    sdmac->peripheral_type == IMX_DMATYPE_ASRC)  in sdma_config_channel()
1173 				sdma_set_watermarklevel_for_p2p(sdmac);  in sdma_config_channel()
1175 			__set_bit(sdmac->event_id0, sdmac->event_mask);  in sdma_config_channel()
1178 		sdmac->shp_addr = sdmac->per_address;  in sdma_config_channel()
1179 		sdmac->per_addr = sdmac->per_address2;  in sdma_config_channel()
1181 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */  in sdma_config_channel()
1184 	ret = sdma_load_context(sdmac);  in sdma_config_channel()
1189 static int sdma_set_channel_priority(struct sdma_channel *sdmac,  in sdma_set_channel_priority()  argument
1192 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_channel_priority()
1193 	int channel = sdmac->channel;  in sdma_set_channel_priority()
1232 	desc->bd = dma_alloc_coherent(desc->sdmac->sdma->dev, bd_size,  in sdma_alloc_bd()
1246 	dma_free_coherent(desc->sdmac->sdma->dev, bd_size, desc->bd,  in sdma_free_bd()
1260 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_alloc_chan_resources()  local
1275 		dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");  in sdma_alloc_chan_resources()
1282 		sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);  in sdma_alloc_chan_resources()
1298 	sdmac->peripheral_type = data->peripheral_type;  in sdma_alloc_chan_resources()
1299 	sdmac->event_id0 = data->dma_request;  in sdma_alloc_chan_resources()
1300 	sdmac->event_id1 = data->dma_request2;  in sdma_alloc_chan_resources()
1302 	ret = clk_enable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1305 	ret = clk_enable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1309 	ret = sdma_set_channel_priority(sdmac, prio);  in sdma_alloc_chan_resources()
1316 	clk_disable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1318 	clk_disable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1324 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_free_chan_resources()  local
1325 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_free_chan_resources()
1331 	if (sdmac->event_id0)  in sdma_free_chan_resources()
1332 		sdma_event_disable(sdmac, sdmac->event_id0);  in sdma_free_chan_resources()
1333 	if (sdmac->event_id1)  in sdma_free_chan_resources()
1334 		sdma_event_disable(sdmac, sdmac->event_id1);  in sdma_free_chan_resources()
1336 	sdmac->event_id0 = 0;  in sdma_free_chan_resources()
1337 	sdmac->event_id1 = 0;  in sdma_free_chan_resources()
1339 	sdma_set_channel_priority(sdmac, 0);  in sdma_free_chan_resources()
1345 static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,  in sdma_transfer_init()  argument
1354 	sdmac->status = DMA_IN_PROGRESS;  in sdma_transfer_init()
1355 	sdmac->direction = direction;  in sdma_transfer_init()
1356 	sdmac->flags = 0;  in sdma_transfer_init()
1362 	desc->sdmac = sdmac;  in sdma_transfer_init()
1370 		sdma_config_ownership(sdmac, false, true, false);  in sdma_transfer_init()
1372 	if (sdma_load_context(sdmac))  in sdma_transfer_init()
1387 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_memcpy()  local
1388 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_memcpy()
1389 	int channel = sdmac->channel;  in sdma_prep_memcpy()
1401 	desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,  in sdma_prep_memcpy()
1436 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_memcpy()
1444 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_slave_sg()  local
1445 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_slave_sg()
1447 	int channel = sdmac->channel;  in sdma_prep_slave_sg()
1451 	sdma_config_write(chan, &sdmac->slave_config, direction);  in sdma_prep_slave_sg()
1453 	desc = sdma_transfer_init(sdmac, direction, sg_len);  in sdma_prep_slave_sg()
1477 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_slave_sg()
1480 		switch (sdmac->word_size) {  in sdma_prep_slave_sg()
1514 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_slave_sg()
1519 	sdmac->status = DMA_ERROR;  in sdma_prep_slave_sg()
1528 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_dma_cyclic()  local
1529 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_dma_cyclic()
1531 	int channel = sdmac->channel;  in sdma_prep_dma_cyclic()
1537 	sdma_config_write(chan, &sdmac->slave_config, direction);  in sdma_prep_dma_cyclic()
1539 	desc = sdma_transfer_init(sdmac, direction, num_periods);  in sdma_prep_dma_cyclic()
1545 	sdmac->flags |= IMX_DMA_SG_LOOP;  in sdma_prep_dma_cyclic()
1561 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1563 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1566 			bd->mode.command = sdmac->word_size;  in sdma_prep_dma_cyclic()
1585 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_dma_cyclic()
1590 	sdmac->status = DMA_ERROR;  in sdma_prep_dma_cyclic()
1598 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config_write()  local
1601 		sdmac->per_address = dmaengine_cfg->src_addr;  in sdma_config_write()
1602 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *  in sdma_config_write()
1604 		sdmac->word_size = dmaengine_cfg->src_addr_width;  in sdma_config_write()
1606 		sdmac->per_address2 = dmaengine_cfg->src_addr;  in sdma_config_write()
1607 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config_write()
1608 		sdmac->watermark_level = dmaengine_cfg->src_maxburst &  in sdma_config_write()
1610 		sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &  in sdma_config_write()
1612 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config_write()
1614 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config_write()
1615 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *  in sdma_config_write()
1617 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config_write()
1619 	sdmac->direction = direction;  in sdma_config_write()
1626 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config()  local
1628 	memcpy(&sdmac->slave_config, dmaengine_cfg, sizeof(*dmaengine_cfg));  in sdma_config()
1631 	if (sdmac->event_id0) {  in sdma_config()
1632 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)  in sdma_config()
1634 		sdma_event_enable(sdmac, sdmac->event_id0);  in sdma_config()
1637 	if (sdmac->event_id1) {  in sdma_config()
1638 		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)  in sdma_config()
1640 		sdma_event_enable(sdmac, sdmac->event_id1);  in sdma_config()
1650 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_tx_status()  local
1661 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_tx_status()
1662 	vd = vchan_find_desc(&sdmac->vc, cookie);  in sdma_tx_status()
1665 		if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tx_status()
1670 	} else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) {  in sdma_tx_status()
1671 		residue = sdmac->desc->chn_count - sdmac->desc->chn_real_count;  in sdma_tx_status()
1675 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_tx_status()
1680 	return sdmac->status;  in sdma_tx_status()
1685 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_issue_pending()  local
1688 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_issue_pending()
1689 	if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)  in sdma_issue_pending()
1690 		sdma_start_desc(sdmac);  in sdma_issue_pending()
1691 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_issue_pending()
1940 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_filter_fn()  local
1946 	sdmac->data = *data;  in sdma_filter_fn()
1947 	chan->private = &sdmac->data;  in sdma_filter_fn()
2068 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_probe()  local
2070 		sdmac->sdma = sdma;  in sdma_probe()
2072 		sdmac->channel = i;  in sdma_probe()
2073 		sdmac->vc.desc_free = sdma_desc_free;  in sdma_probe()
2074 		INIT_WORK(&sdmac->terminate_worker,  in sdma_probe()
2082 			vchan_init(&sdmac->vc, &sdma->dma_device);  in sdma_probe()
2194 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_remove()  local
2196 		tasklet_kill(&sdmac->vc.task);  in sdma_remove()
2197 		sdma_free_chan_resources(&sdmac->vc.chan);  in sdma_remove()