33 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
605 static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_enable_channel()  argument
608 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_enable_channel()
616 	ichan->active_buffer = 0;  in ipu_enable_channel()
617 	ichan->status = IPU_CHANNEL_ENABLED;  in ipu_enable_channel()
653 static int ipu_init_channel_buffer(struct idmac_channel *ichan,  in ipu_init_channel_buffer()  argument
660 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel_buffer()
661 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in ipu_init_channel_buffer()
701 	ichan->status = IPU_CHANNEL_READY;  in ipu_init_channel_buffer()
732 static void ipu_update_channel_buffer(struct idmac_channel *ichan,  in ipu_update_channel_buffer()  argument
735 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_update_channel_buffer()
745 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
756 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
776 static int ipu_submit_buffer(struct idmac_channel *ichan,  in ipu_submit_buffer()  argument
779 	unsigned int chan_id = ichan->dma_chan.chan_id;  in ipu_submit_buffer()
780 	struct device *dev = &ichan->dma_chan.dev->device;  in ipu_submit_buffer()
791 	ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));  in ipu_submit_buffer()
801 static int ipu_submit_channel_buffers(struct idmac_channel *ichan,  in ipu_submit_channel_buffers()  argument
808 		if (!ichan->sg[i]) {  in ipu_submit_channel_buffers()
809 			ichan->sg[i] = sg;  in ipu_submit_channel_buffers()
811 			ret = ipu_submit_buffer(ichan, desc, sg, i);  in ipu_submit_channel_buffers()
825 	struct idmac_channel *ichan = to_idmac_chan(tx->chan);  in idmac_tx_submit()  local
828 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_tx_submit()
840 	mutex_lock(&ichan->chan_mutex);  in idmac_tx_submit()
844 	if (ichan->status < IPU_CHANNEL_READY) {  in idmac_tx_submit()
845 		struct idmac_video_param *video = &ichan->params.video;  in idmac_tx_submit()
853 		WARN_ON(ichan->sg[0] || ichan->sg[1]);  in idmac_tx_submit()
855 		cookie = ipu_init_channel_buffer(ichan,  in idmac_tx_submit()
872 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
874 	list_add_tail(&desc->list, &ichan->queue);  in idmac_tx_submit()
876 	ret = ipu_submit_channel_buffers(ichan, desc);  in idmac_tx_submit()
878 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
885 	if (ichan->status < IPU_CHANNEL_ENABLED) {  in idmac_tx_submit()
886 		ret = ipu_enable_channel(idmac, ichan);  in idmac_tx_submit()
897 		spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
899 		spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
901 		ichan->dma_chan.cookie = cookie;  in idmac_tx_submit()
905 	mutex_unlock(&ichan->chan_mutex);  in idmac_tx_submit()
911 static int idmac_desc_alloc(struct idmac_channel *ichan, int n)  in idmac_desc_alloc()  argument
915 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in idmac_desc_alloc()
923 	ichan->n_tx_desc = n;  in idmac_desc_alloc()
924 	ichan->desc = desc;  in idmac_desc_alloc()
925 	INIT_LIST_HEAD(&ichan->queue);  in idmac_desc_alloc()
926 	INIT_LIST_HEAD(&ichan->free_list);  in idmac_desc_alloc()
932 		dma_async_tx_descriptor_init(txd, &ichan->dma_chan);  in idmac_desc_alloc()
935 		list_add(&desc->list, &ichan->free_list);  in idmac_desc_alloc()
951 static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_init_channel()  argument
953 	union ipu_channel_param *params = &ichan->params;  in ipu_init_channel()
955 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel()
997 	if (n_desc && !ichan->desc)  in ipu_init_channel()
998 		ret = idmac_desc_alloc(ichan, n_desc);  in ipu_init_channel()
1010 static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_uninit_channel()  argument
1012 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_uninit_channel()
1032 	ichan->sec_chan_en = false;  in ipu_uninit_channel()
1059 	ichan->n_tx_desc = 0;  in ipu_uninit_channel()
1060 	vfree(ichan->desc);  in ipu_uninit_channel()
1061 	ichan->desc = NULL;  in ipu_uninit_channel()
1072 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,  in ipu_disable_channel()  argument
1075 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_disable_channel()
1108 		     timeout && !ipu_irq_status(ichan->eof_irq); timeout--)  in ipu_disable_channel()
1126 static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,  in idmac_sg_next()  argument
1135 	if ((*desc)->list.next == &ichan->queue)  in idmac_sg_next()
1158 	struct idmac_channel *ichan = dev_id;  in idmac_interrupt()  local
1159 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_interrupt()
1160 	unsigned int chan_id = ichan->dma_chan.chan_id;  in idmac_interrupt()
1171 	dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);  in idmac_interrupt()
1197 	spin_lock(&ichan->lock);  in idmac_interrupt()
1198 	if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||  in idmac_interrupt()
1199 		     (!ichan->active_buffer && (ready0 >> chan_id) & 1)  in idmac_interrupt()
1201 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1205 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1209 	if (unlikely(list_empty(&ichan->queue))) {  in idmac_interrupt()
1210 		ichan->sg[ichan->active_buffer] = NULL;  in idmac_interrupt()
1211 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1215 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1224 	sg = &ichan->sg[ichan->active_buffer];  in idmac_interrupt()
1225 	sgnext = ichan->sg[!ichan->active_buffer];  in idmac_interrupt()
1228 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1232 	desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);  in idmac_interrupt()
1238 		ichan->active_buffer, curbuf);  in idmac_interrupt()
1241 	sgnew = idmac_sg_next(ichan, &descnew, *sg);  in idmac_interrupt()
1252 		ichan->sg[!ichan->active_buffer] = sgnew;  in idmac_interrupt()
1255 			ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);  in idmac_interrupt()
1260 			ichan->status = IPU_CHANNEL_READY;  in idmac_interrupt()
1266 	sgnew = idmac_sg_next(ichan, &descnew, sgnew);  in idmac_interrupt()
1280 	    ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {  in idmac_interrupt()
1284 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1287 		spin_lock(&ichan->lock);  in idmac_interrupt()
1291 	ichan->active_buffer = !ichan->active_buffer;  in idmac_interrupt()
1297 	spin_unlock(&ichan->lock);  in idmac_interrupt()
1311 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_gc_tasklet()  local
1317 		for (j = 0; j < ichan->n_tx_desc; j++) {  in ipu_gc_tasklet()
1318 			desc = ichan->desc + j;  in ipu_gc_tasklet()
1319 			spin_lock_irqsave(&ichan->lock, flags);  in ipu_gc_tasklet()
1321 				list_move(&desc->list, &ichan->free_list);  in ipu_gc_tasklet()
1323 					if (ichan->sg[0] == sg)  in ipu_gc_tasklet()
1324 						ichan->sg[0] = NULL;  in ipu_gc_tasklet()
1325 					else if (ichan->sg[1] == sg)  in ipu_gc_tasklet()
1326 						ichan->sg[1] = NULL;  in ipu_gc_tasklet()
1330 			spin_unlock_irqrestore(&ichan->lock, flags);  in ipu_gc_tasklet()
1341 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_prep_slave_sg()  local
1356 	mutex_lock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1358 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_prep_slave_sg()
1359 	if (!list_empty(&ichan->free_list)) {  in idmac_prep_slave_sg()
1360 		desc = list_entry(ichan->free_list.next,  in idmac_prep_slave_sg()
1370 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_prep_slave_sg()
1372 	mutex_unlock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1382 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_issue_pending()  local
1389 	ipu_select_buffer(chan->chan_id, ichan->active_buffer);  in idmac_issue_pending()
1402 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_pause()  local
1408 	mutex_lock(&ichan->chan_mutex);  in idmac_pause()
1414 	list_for_each_safe(list, tmp, &ichan->queue)  in idmac_pause()
1417 	ichan->sg[0] = NULL;  in idmac_pause()
1418 	ichan->sg[1] = NULL;  in idmac_pause()
1422 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_pause()
1424 	mutex_unlock(&ichan->chan_mutex);  in idmac_pause()
1431 	struct idmac_channel *ichan = to_idmac_chan(chan);  in __idmac_terminate_all()  local
1437 	ipu_disable_channel(idmac, ichan,  in __idmac_terminate_all()
1438 			    ichan->status >= IPU_CHANNEL_ENABLED);  in __idmac_terminate_all()
1443 	spin_lock_irqsave(&ichan->lock, flags);  in __idmac_terminate_all()
1444 	list_splice_init(&ichan->queue, &ichan->free_list);  in __idmac_terminate_all()
1446 	if (ichan->desc)  in __idmac_terminate_all()
1447 		for (i = 0; i < ichan->n_tx_desc; i++) {  in __idmac_terminate_all()
1448 			struct idmac_tx_desc *desc = ichan->desc + i;  in __idmac_terminate_all()
1451 				list_add(&desc->list, &ichan->free_list);  in __idmac_terminate_all()
1456 	ichan->sg[0] = NULL;  in __idmac_terminate_all()
1457 	ichan->sg[1] = NULL;  in __idmac_terminate_all()
1458 	spin_unlock_irqrestore(&ichan->lock, flags);  in __idmac_terminate_all()
1462 	ichan->status = IPU_CHANNEL_INITIALIZED;  in __idmac_terminate_all()
1469 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_terminate_all()  local
1472 	mutex_lock(&ichan->chan_mutex);  in idmac_terminate_all()
1476 	mutex_unlock(&ichan->chan_mutex);  in idmac_terminate_all()
1484 	struct idmac_channel *ichan = dev_id;  in ic_sof_irq()  local
1486 	       irq, ichan->dma_chan.chan_id);  in ic_sof_irq()
1493 	struct idmac_channel *ichan = dev_id;  in ic_eof_irq()  local
1495 	       irq, ichan->dma_chan.chan_id);  in ic_eof_irq()
1505 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_alloc_chan_resources()  local
1511 	WARN_ON(ichan->status != IPU_CHANNEL_FREE);  in idmac_alloc_chan_resources()
1519 	ichan->eof_irq = ret;  in idmac_alloc_chan_resources()
1525 	ipu_disable_channel(idmac, ichan, true);  in idmac_alloc_chan_resources()
1527 	ret = ipu_init_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1531 	ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,  in idmac_alloc_chan_resources()
1532 			  ichan->eof_name, ichan);  in idmac_alloc_chan_resources()
1540 			ret = request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);  in idmac_alloc_chan_resources()
1546 			ret = request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);  in idmac_alloc_chan_resources()
1553 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_alloc_chan_resources()
1556 		chan->chan_id, ichan->eof_irq);  in idmac_alloc_chan_resources()
1561 	ipu_uninit_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1570 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_free_chan_resources()  local
1573 	mutex_lock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1577 	if (ichan->status > IPU_CHANNEL_FREE) {  in idmac_free_chan_resources()
1581 				free_irq(ic_sof, ichan);  in idmac_free_chan_resources()
1586 				free_irq(ic_eof, ichan);  in idmac_free_chan_resources()
1592 		free_irq(ichan->eof_irq, ichan);  in idmac_free_chan_resources()
1596 	ichan->status = IPU_CHANNEL_FREE;  in idmac_free_chan_resources()
1598 	ipu_uninit_channel(idmac, ichan);  in idmac_free_chan_resources()
1600 	mutex_unlock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1634 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_init()  local
1635 		struct dma_chan *dma_chan = &ichan->dma_chan;  in ipu_idmac_init()
1637 		spin_lock_init(&ichan->lock);  in ipu_idmac_init()
1638 		mutex_init(&ichan->chan_mutex);  in ipu_idmac_init()
1640 		ichan->status		= IPU_CHANNEL_FREE;  in ipu_idmac_init()
1641 		ichan->sec_chan_en	= false;  in ipu_idmac_init()
1642 		snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);  in ipu_idmac_init()
1661 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_exit()  local
1663 		idmac_terminate_all(&ichan->dma_chan);  in ipu_idmac_exit()