326 	struct sdma_channel	*sdmac;  member
602 static int sdma_config_ownership(struct sdma_channel *sdmac,  in sdma_config_ownership()  argument
605 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_config_ownership()
606 	int channel = sdmac->channel;  in sdma_config_ownership()
700 static void sdma_event_enable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_enable()  argument
702 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_enable()
703 	int channel = sdmac->channel;  in sdma_event_enable()
712 static void sdma_event_disable(struct sdma_channel *sdmac, unsigned int event)  in sdma_event_disable()  argument
714 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_event_disable()
715 	int channel = sdmac->channel;  in sdma_event_disable()
729 static void sdma_start_desc(struct sdma_channel *sdmac)  in sdma_start_desc()  argument
731 	struct virt_dma_desc *vd = vchan_next_desc(&sdmac->vc);  in sdma_start_desc()
733 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_start_desc()
734 	int channel = sdmac->channel;  in sdma_start_desc()
737 		sdmac->desc = NULL;  in sdma_start_desc()
740 	sdmac->desc = desc = to_sdma_desc(&vd->tx);  in sdma_start_desc()
745 	if (!(sdmac->flags & IMX_DMA_SG_LOOP))  in sdma_start_desc()
750 	sdma_enable_channel(sdma, sdmac->channel);  in sdma_start_desc()
753 static void sdma_update_channel_loop(struct sdma_channel *sdmac)  in sdma_update_channel_loop()  argument
757 	enum dma_status	old_status = sdmac->status;  in sdma_update_channel_loop()
763 	while (sdmac->desc) {  in sdma_update_channel_loop()
764 		struct sdma_desc *desc = sdmac->desc;  in sdma_update_channel_loop()
773 			sdmac->status = DMA_ERROR;  in sdma_update_channel_loop()
794 		spin_unlock(&sdmac->vc.lock);  in sdma_update_channel_loop()
796 		spin_lock(&sdmac->vc.lock);  in sdma_update_channel_loop()
799 			sdmac->status = old_status;  in sdma_update_channel_loop()
805 	struct sdma_channel *sdmac = (struct sdma_channel *) data;  in mxc_sdma_handle_channel_normal()  local
809 	sdmac->desc->chn_real_count = 0;  in mxc_sdma_handle_channel_normal()
814 	for (i = 0; i < sdmac->desc->num_bd; i++) {  in mxc_sdma_handle_channel_normal()
815 		bd = &sdmac->desc->bd[i];  in mxc_sdma_handle_channel_normal()
819 		 sdmac->desc->chn_real_count += bd->mode.count;  in mxc_sdma_handle_channel_normal()
823 		sdmac->status = DMA_ERROR;  in mxc_sdma_handle_channel_normal()
825 		sdmac->status = DMA_COMPLETE;  in mxc_sdma_handle_channel_normal()
840 		struct sdma_channel *sdmac = &sdma->channel[channel];  in sdma_int_handler()  local
843 		spin_lock(&sdmac->vc.lock);  in sdma_int_handler()
844 		desc = sdmac->desc;  in sdma_int_handler()
846 			if (sdmac->flags & IMX_DMA_SG_LOOP) {  in sdma_int_handler()
847 				sdma_update_channel_loop(sdmac);  in sdma_int_handler()
849 				mxc_sdma_handle_channel_normal(sdmac);  in sdma_int_handler()
851 				sdma_start_desc(sdmac);  in sdma_int_handler()
855 		spin_unlock(&sdmac->vc.lock);  in sdma_int_handler()
865 static void sdma_get_pc(struct sdma_channel *sdmac,  in sdma_get_pc()  argument
868 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_get_pc()
876 	sdmac->pc_from_device = 0;  in sdma_get_pc()
877 	sdmac->pc_to_device = 0;  in sdma_get_pc()
878 	sdmac->device_to_device = 0;  in sdma_get_pc()
879 	sdmac->pc_to_pc = 0;  in sdma_get_pc()
953 	sdmac->pc_from_device = per_2_emi;  in sdma_get_pc()
954 	sdmac->pc_to_device = emi_2_per;  in sdma_get_pc()
955 	sdmac->device_to_device = per_2_per;  in sdma_get_pc()
956 	sdmac->pc_to_pc = emi_2_emi;  in sdma_get_pc()
959 static int sdma_load_context(struct sdma_channel *sdmac)  in sdma_load_context()  argument
961 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_load_context()
962 	int channel = sdmac->channel;  in sdma_load_context()
969 	if (sdmac->direction == DMA_DEV_TO_MEM)  in sdma_load_context()
970 		load_address = sdmac->pc_from_device;  in sdma_load_context()
971 	else if (sdmac->direction == DMA_DEV_TO_DEV)  in sdma_load_context()
972 		load_address = sdmac->device_to_device;  in sdma_load_context()
973 	else if (sdmac->direction == DMA_MEM_TO_MEM)  in sdma_load_context()
974 		load_address = sdmac->pc_to_pc;  in sdma_load_context()
976 		load_address = sdmac->pc_to_device;  in sdma_load_context()
982 	dev_dbg(sdma->dev, "wml = 0x%08x\n", (u32)sdmac->watermark_level);  in sdma_load_context()
983 	dev_dbg(sdma->dev, "shp_addr = 0x%08x\n", sdmac->shp_addr);  in sdma_load_context()
984 	dev_dbg(sdma->dev, "per_addr = 0x%08x\n", sdmac->per_addr);  in sdma_load_context()
985 	dev_dbg(sdma->dev, "event_mask0 = 0x%08x\n", (u32)sdmac->event_mask[0]);  in sdma_load_context()
986 	dev_dbg(sdma->dev, "event_mask1 = 0x%08x\n", (u32)sdmac->event_mask[1]);  in sdma_load_context()
996 	context->gReg[0] = sdmac->event_mask[1];  in sdma_load_context()
997 	context->gReg[1] = sdmac->event_mask[0];  in sdma_load_context()
998 	context->gReg[2] = sdmac->per_addr;  in sdma_load_context()
999 	context->gReg[6] = sdmac->shp_addr;  in sdma_load_context()
1000 	context->gReg[7] = sdmac->watermark_level;  in sdma_load_context()
1021 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel()  local
1022 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_disable_channel()
1023 	int channel = sdmac->channel;  in sdma_disable_channel()
1026 	sdmac->status = DMA_ERROR;  in sdma_disable_channel()
1033 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_disable_channel_with_delay()  local
1038 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_disable_channel_with_delay()
1039 	vchan_get_all_descriptors(&sdmac->vc, &head);  in sdma_disable_channel_with_delay()
1040 	sdmac->desc = NULL;  in sdma_disable_channel_with_delay()
1041 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_disable_channel_with_delay()
1042 	vchan_dma_desc_free_list(&sdmac->vc, &head);  in sdma_disable_channel_with_delay()
1055 static void sdma_set_watermarklevel_for_p2p(struct sdma_channel *sdmac)  in sdma_set_watermarklevel_for_p2p()  argument
1057 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_watermarklevel_for_p2p()
1059 	int lwml = sdmac->watermark_level & SDMA_WATERMARK_LEVEL_LWML;  in sdma_set_watermarklevel_for_p2p()
1060 	int hwml = (sdmac->watermark_level & SDMA_WATERMARK_LEVEL_HWML) >> 16;  in sdma_set_watermarklevel_for_p2p()
1062 	set_bit(sdmac->event_id0 % 32, &sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
1063 	set_bit(sdmac->event_id1 % 32, &sdmac->event_mask[0]);  in sdma_set_watermarklevel_for_p2p()
1065 	if (sdmac->event_id0 > 31)  in sdma_set_watermarklevel_for_p2p()
1066 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_LWE;  in sdma_set_watermarklevel_for_p2p()
1068 	if (sdmac->event_id1 > 31)  in sdma_set_watermarklevel_for_p2p()
1069 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_HWE;  in sdma_set_watermarklevel_for_p2p()
1077 		sdmac->watermark_level &= ~(SDMA_WATERMARK_LEVEL_LWML |  in sdma_set_watermarklevel_for_p2p()
1079 		sdmac->watermark_level |= hwml;  in sdma_set_watermarklevel_for_p2p()
1080 		sdmac->watermark_level |= lwml << 16;  in sdma_set_watermarklevel_for_p2p()
1081 		swap(sdmac->event_mask[0], sdmac->event_mask[1]);  in sdma_set_watermarklevel_for_p2p()
1084 	if (sdmac->per_address2 >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
1085 			sdmac->per_address2 <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
1086 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_SP;  in sdma_set_watermarklevel_for_p2p()
1088 	if (sdmac->per_address >= sdma->spba_start_addr &&  in sdma_set_watermarklevel_for_p2p()
1089 			sdmac->per_address <= sdma->spba_end_addr)  in sdma_set_watermarklevel_for_p2p()
1090 		sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_DP;  in sdma_set_watermarklevel_for_p2p()
1092 	sdmac->watermark_level |= SDMA_WATERMARK_LEVEL_CONT;  in sdma_set_watermarklevel_for_p2p()
1097 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config_channel()  local
1102 	sdmac->event_mask[0] = 0;  in sdma_config_channel()
1103 	sdmac->event_mask[1] = 0;  in sdma_config_channel()
1104 	sdmac->shp_addr = 0;  in sdma_config_channel()
1105 	sdmac->per_addr = 0;  in sdma_config_channel()
1107 	if (sdmac->event_id0) {  in sdma_config_channel()
1108 		if (sdmac->event_id0 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
1110 		sdma_event_enable(sdmac, sdmac->event_id0);  in sdma_config_channel()
1113 	if (sdmac->event_id1) {  in sdma_config_channel()
1114 		if (sdmac->event_id1 >= sdmac->sdma->drvdata->num_events)  in sdma_config_channel()
1116 		sdma_event_enable(sdmac, sdmac->event_id1);  in sdma_config_channel()
1119 	switch (sdmac->peripheral_type) {  in sdma_config_channel()
1121 		sdma_config_ownership(sdmac, false, true, true);  in sdma_config_channel()
1124 		sdma_config_ownership(sdmac, false, true, false);  in sdma_config_channel()
1127 		sdma_config_ownership(sdmac, true, true, false);  in sdma_config_channel()
1131 	sdma_get_pc(sdmac, sdmac->peripheral_type);  in sdma_config_channel()
1133 	if ((sdmac->peripheral_type != IMX_DMATYPE_MEMORY) &&  in sdma_config_channel()
1134 			(sdmac->peripheral_type != IMX_DMATYPE_DSP)) {  in sdma_config_channel()
1136 		if (sdmac->event_id1) {  in sdma_config_channel()
1137 			if (sdmac->peripheral_type == IMX_DMATYPE_ASRC_SP ||  in sdma_config_channel()
1138 			    sdmac->peripheral_type == IMX_DMATYPE_ASRC)  in sdma_config_channel()
1139 				sdma_set_watermarklevel_for_p2p(sdmac);  in sdma_config_channel()
1141 			__set_bit(sdmac->event_id0, sdmac->event_mask);  in sdma_config_channel()
1144 		sdmac->shp_addr = sdmac->per_address;  in sdma_config_channel()
1145 		sdmac->per_addr = sdmac->per_address2;  in sdma_config_channel()
1147 		sdmac->watermark_level = 0; /* FIXME: M3_BASE_ADDRESS */  in sdma_config_channel()
1150 	ret = sdma_load_context(sdmac);  in sdma_config_channel()
1155 static int sdma_set_channel_priority(struct sdma_channel *sdmac,  in sdma_set_channel_priority()  argument
1158 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_set_channel_priority()
1159 	int channel = sdmac->channel;  in sdma_set_channel_priority()
1197 	desc->bd = dma_pool_alloc(desc->sdmac->bd_pool, GFP_NOWAIT,  in sdma_alloc_bd()
1209 	dma_pool_free(desc->sdmac->bd_pool, desc->bd, desc->bd_phys);  in sdma_free_bd()
1222 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_alloc_chan_resources()  local
1237 		dev_dbg(sdmac->sdma->dev, "MEMCPY in case?\n");  in sdma_alloc_chan_resources()
1244 		sdma_get_pc(sdmac, IMX_DMATYPE_MEMORY);  in sdma_alloc_chan_resources()
1260 	sdmac->peripheral_type = data->peripheral_type;  in sdma_alloc_chan_resources()
1261 	sdmac->event_id0 = data->dma_request;  in sdma_alloc_chan_resources()
1262 	sdmac->event_id1 = data->dma_request2;  in sdma_alloc_chan_resources()
1264 	ret = clk_enable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1267 	ret = clk_enable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1271 	ret = sdma_set_channel_priority(sdmac, prio);  in sdma_alloc_chan_resources()
1275 	sdmac->bd_pool = dma_pool_create("bd_pool", chan->device->dev,  in sdma_alloc_chan_resources()
1282 	clk_disable(sdmac->sdma->clk_ahb);  in sdma_alloc_chan_resources()
1284 	clk_disable(sdmac->sdma->clk_ipg);  in sdma_alloc_chan_resources()
1290 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_free_chan_resources()  local
1291 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_free_chan_resources()
1295 	if (sdmac->event_id0)  in sdma_free_chan_resources()
1296 		sdma_event_disable(sdmac, sdmac->event_id0);  in sdma_free_chan_resources()
1297 	if (sdmac->event_id1)  in sdma_free_chan_resources()
1298 		sdma_event_disable(sdmac, sdmac->event_id1);  in sdma_free_chan_resources()
1300 	sdmac->event_id0 = 0;  in sdma_free_chan_resources()
1301 	sdmac->event_id1 = 0;  in sdma_free_chan_resources()
1303 	sdma_set_channel_priority(sdmac, 0);  in sdma_free_chan_resources()
1308 	dma_pool_destroy(sdmac->bd_pool);  in sdma_free_chan_resources()
1309 	sdmac->bd_pool = NULL;  in sdma_free_chan_resources()
1312 static struct sdma_desc *sdma_transfer_init(struct sdma_channel *sdmac,  in sdma_transfer_init()  argument
1321 	sdmac->status = DMA_IN_PROGRESS;  in sdma_transfer_init()
1322 	sdmac->direction = direction;  in sdma_transfer_init()
1323 	sdmac->flags = 0;  in sdma_transfer_init()
1329 	desc->sdmac = sdmac;  in sdma_transfer_init()
1337 		sdma_config_ownership(sdmac, false, true, false);  in sdma_transfer_init()
1339 	if (sdma_load_context(sdmac))  in sdma_transfer_init()
1354 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_memcpy()  local
1355 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_memcpy()
1356 	int channel = sdmac->channel;  in sdma_prep_memcpy()
1368 	desc = sdma_transfer_init(sdmac, DMA_MEM_TO_MEM,  in sdma_prep_memcpy()
1403 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_memcpy()
1411 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_slave_sg()  local
1412 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_slave_sg()
1414 	int channel = sdmac->channel;  in sdma_prep_slave_sg()
1418 	desc = sdma_transfer_init(sdmac, direction, sg_len);  in sdma_prep_slave_sg()
1442 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_slave_sg()
1445 		switch (sdmac->word_size) {  in sdma_prep_slave_sg()
1479 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_slave_sg()
1484 	sdmac->status = DMA_ERROR;  in sdma_prep_slave_sg()
1493 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_prep_dma_cyclic()  local
1494 	struct sdma_engine *sdma = sdmac->sdma;  in sdma_prep_dma_cyclic()
1496 	int channel = sdmac->channel;  in sdma_prep_dma_cyclic()
1502 	desc = sdma_transfer_init(sdmac, direction, num_periods);  in sdma_prep_dma_cyclic()
1508 	sdmac->flags |= IMX_DMA_SG_LOOP;  in sdma_prep_dma_cyclic()
1524 		if (sdmac->word_size > DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1526 		if (sdmac->word_size == DMA_SLAVE_BUSWIDTH_4_BYTES)  in sdma_prep_dma_cyclic()
1529 			bd->mode.command = sdmac->word_size;  in sdma_prep_dma_cyclic()
1548 	return vchan_tx_prep(&sdmac->vc, &desc->vd, flags);  in sdma_prep_dma_cyclic()
1553 	sdmac->status = DMA_ERROR;  in sdma_prep_dma_cyclic()
1560 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_config()  local
1563 		sdmac->per_address = dmaengine_cfg->src_addr;  in sdma_config()
1564 		sdmac->watermark_level = dmaengine_cfg->src_maxburst *  in sdma_config()
1566 		sdmac->word_size = dmaengine_cfg->src_addr_width;  in sdma_config()
1568 		sdmac->per_address2 = dmaengine_cfg->src_addr;  in sdma_config()
1569 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config()
1570 		sdmac->watermark_level = dmaengine_cfg->src_maxburst &  in sdma_config()
1572 		sdmac->watermark_level |= (dmaengine_cfg->dst_maxburst << 16) &  in sdma_config()
1574 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config()
1576 		sdmac->per_address = dmaengine_cfg->dst_addr;  in sdma_config()
1577 		sdmac->watermark_level = dmaengine_cfg->dst_maxburst *  in sdma_config()
1579 		sdmac->word_size = dmaengine_cfg->dst_addr_width;  in sdma_config()
1581 	sdmac->direction = dmaengine_cfg->direction;  in sdma_config()
1589 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_tx_status()  local
1600 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_tx_status()
1601 	vd = vchan_find_desc(&sdmac->vc, cookie);  in sdma_tx_status()
1604 		if (sdmac->flags & IMX_DMA_SG_LOOP)  in sdma_tx_status()
1609 	} else if (sdmac->desc && sdmac->desc->vd.tx.cookie == cookie) {  in sdma_tx_status()
1610 		residue = sdmac->desc->chn_count - sdmac->desc->chn_real_count;  in sdma_tx_status()
1614 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_tx_status()
1619 	return sdmac->status;  in sdma_tx_status()
1624 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_issue_pending()  local
1627 	spin_lock_irqsave(&sdmac->vc.lock, flags);  in sdma_issue_pending()
1628 	if (vchan_issue_pending(&sdmac->vc) && !sdmac->desc)  in sdma_issue_pending()
1629 		sdma_start_desc(sdmac);  in sdma_issue_pending()
1630 	spin_unlock_irqrestore(&sdmac->vc.lock, flags);  in sdma_issue_pending()
1869 	struct sdma_channel *sdmac = to_sdma_chan(chan);  in sdma_filter_fn()  local
1875 	sdmac->data = *data;  in sdma_filter_fn()
1876 	chan->private = &sdmac->data;  in sdma_filter_fn()
1996 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_probe()  local
1998 		sdmac->sdma = sdma;  in sdma_probe()
2000 		sdmac->channel = i;  in sdma_probe()
2001 		sdmac->vc.desc_free = sdma_desc_free;  in sdma_probe()
2008 			vchan_init(&sdmac->vc, &sdma->dma_device);  in sdma_probe()
2112 		struct sdma_channel *sdmac = &sdma->channel[i];  in sdma_remove()  local
2114 		tasklet_kill(&sdmac->vc.task);  in sdma_remove()
2115 		sdma_free_chan_resources(&sdmac->vc.chan);  in sdma_remove()