30 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,
602 static int ipu_enable_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_enable_channel()  argument
605 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_enable_channel()
613 	ichan->active_buffer = 0;  in ipu_enable_channel()
614 	ichan->status = IPU_CHANNEL_ENABLED;  in ipu_enable_channel()
651 static int ipu_init_channel_buffer(struct idmac_channel *ichan,  in ipu_init_channel_buffer()  argument
658 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel_buffer()
659 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in ipu_init_channel_buffer()
699 	ichan->status = IPU_CHANNEL_READY;  in ipu_init_channel_buffer()
730 static void ipu_update_channel_buffer(struct idmac_channel *ichan,  in ipu_update_channel_buffer()  argument
733 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_update_channel_buffer()
743 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
754 			ichan->status = IPU_CHANNEL_READY;  in ipu_update_channel_buffer()
774 static int ipu_submit_buffer(struct idmac_channel *ichan,  in ipu_submit_buffer()  argument
777 	unsigned int chan_id = ichan->dma_chan.chan_id;  in ipu_submit_buffer()
778 	struct device *dev = &ichan->dma_chan.dev->device;  in ipu_submit_buffer()
789 	ipu_update_channel_buffer(ichan, buf_idx, sg_dma_address(sg));  in ipu_submit_buffer()
799 static int ipu_submit_channel_buffers(struct idmac_channel *ichan,  in ipu_submit_channel_buffers()  argument
806 		if (!ichan->sg[i]) {  in ipu_submit_channel_buffers()
807 			ichan->sg[i] = sg;  in ipu_submit_channel_buffers()
809 			ret = ipu_submit_buffer(ichan, desc, sg, i);  in ipu_submit_channel_buffers()
823 	struct idmac_channel *ichan = to_idmac_chan(tx->chan);  in idmac_tx_submit()  local
826 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_tx_submit()
838 	mutex_lock(&ichan->chan_mutex);  in idmac_tx_submit()
842 	if (ichan->status < IPU_CHANNEL_READY) {  in idmac_tx_submit()
843 		struct idmac_video_param *video = &ichan->params.video;  in idmac_tx_submit()
851 		WARN_ON(ichan->sg[0] || ichan->sg[1]);  in idmac_tx_submit()
853 		cookie = ipu_init_channel_buffer(ichan,  in idmac_tx_submit()
870 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
872 	list_add_tail(&desc->list, &ichan->queue);  in idmac_tx_submit()
874 	ret = ipu_submit_channel_buffers(ichan, desc);  in idmac_tx_submit()
876 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
883 	if (ichan->status < IPU_CHANNEL_ENABLED) {  in idmac_tx_submit()
884 		ret = ipu_enable_channel(idmac, ichan);  in idmac_tx_submit()
895 		spin_lock_irqsave(&ichan->lock, flags);  in idmac_tx_submit()
897 		spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_tx_submit()
899 		ichan->dma_chan.cookie = cookie;  in idmac_tx_submit()
903 	mutex_unlock(&ichan->chan_mutex);  in idmac_tx_submit()
909 static int idmac_desc_alloc(struct idmac_channel *ichan, int n)  in idmac_desc_alloc()  argument
913 	struct idmac *idmac = to_idmac(ichan->dma_chan.device);  in idmac_desc_alloc()
921 	ichan->n_tx_desc = n;  in idmac_desc_alloc()
922 	ichan->desc = desc;  in idmac_desc_alloc()
923 	INIT_LIST_HEAD(&ichan->queue);  in idmac_desc_alloc()
924 	INIT_LIST_HEAD(&ichan->free_list);  in idmac_desc_alloc()
930 		dma_async_tx_descriptor_init(txd, &ichan->dma_chan);  in idmac_desc_alloc()
933 		list_add(&desc->list, &ichan->free_list);  in idmac_desc_alloc()
949 static int ipu_init_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_init_channel()  argument
951 	union ipu_channel_param *params = &ichan->params;  in ipu_init_channel()
953 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_init_channel()
996 	if (n_desc && !ichan->desc)  in ipu_init_channel()
997 		ret = idmac_desc_alloc(ichan, n_desc);  in ipu_init_channel()
1009 static void ipu_uninit_channel(struct idmac *idmac, struct idmac_channel *ichan)  in ipu_uninit_channel()  argument
1011 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_uninit_channel()
1031 	ichan->sec_chan_en = false;  in ipu_uninit_channel()
1058 	ichan->n_tx_desc = 0;  in ipu_uninit_channel()
1059 	vfree(ichan->desc);  in ipu_uninit_channel()
1060 	ichan->desc = NULL;  in ipu_uninit_channel()
1071 static int ipu_disable_channel(struct idmac *idmac, struct idmac_channel *ichan,  in ipu_disable_channel()  argument
1074 	enum ipu_channel channel = ichan->dma_chan.chan_id;  in ipu_disable_channel()
1107 		     timeout && !ipu_irq_status(ichan->eof_irq); timeout--)  in ipu_disable_channel()
1125 static struct scatterlist *idmac_sg_next(struct idmac_channel *ichan,  in idmac_sg_next()  argument
1134 	if ((*desc)->list.next == &ichan->queue)  in idmac_sg_next()
1157 	struct idmac_channel *ichan = dev_id;  in idmac_interrupt()  local
1158 	struct device *dev = &ichan->dma_chan.dev->device;  in idmac_interrupt()
1159 	unsigned int chan_id = ichan->dma_chan.chan_id;  in idmac_interrupt()
1169 	dev_dbg(dev, "IDMAC irq %d, buf %d\n", irq, ichan->active_buffer);  in idmac_interrupt()
1195 	spin_lock(&ichan->lock);  in idmac_interrupt()
1196 	if (unlikely((ichan->active_buffer && (ready1 >> chan_id) & 1) ||  in idmac_interrupt()
1197 		     (!ichan->active_buffer && (ready0 >> chan_id) & 1)  in idmac_interrupt()
1199 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1203 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1207 	if (unlikely(list_empty(&ichan->queue))) {  in idmac_interrupt()
1208 		ichan->sg[ichan->active_buffer] = NULL;  in idmac_interrupt()
1209 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1213 			ichan->active_buffer, ready0, ready1);  in idmac_interrupt()
1222 	sg = &ichan->sg[ichan->active_buffer];  in idmac_interrupt()
1223 	sgnext = ichan->sg[!ichan->active_buffer];  in idmac_interrupt()
1226 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1230 	desc = list_entry(ichan->queue.next, struct idmac_tx_desc, list);  in idmac_interrupt()
1236 		ichan->active_buffer, curbuf);  in idmac_interrupt()
1239 	sgnew = idmac_sg_next(ichan, &descnew, *sg);  in idmac_interrupt()
1250 		ichan->sg[!ichan->active_buffer] = sgnew;  in idmac_interrupt()
1253 			ipu_submit_buffer(ichan, descnew, sgnew, !ichan->active_buffer);  in idmac_interrupt()
1258 			ichan->status = IPU_CHANNEL_READY;  in idmac_interrupt()
1264 	sgnew = idmac_sg_next(ichan, &descnew, sgnew);  in idmac_interrupt()
1278 	    ipu_submit_buffer(ichan, descnew, sgnew, ichan->active_buffer) < 0) {  in idmac_interrupt()
1282 		spin_unlock(&ichan->lock);  in idmac_interrupt()
1285 		spin_lock(&ichan->lock);  in idmac_interrupt()
1289 	ichan->active_buffer = !ichan->active_buffer;  in idmac_interrupt()
1295 	spin_unlock(&ichan->lock);  in idmac_interrupt()
1309 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_gc_tasklet()  local
1315 		for (j = 0; j < ichan->n_tx_desc; j++) {  in ipu_gc_tasklet()
1316 			desc = ichan->desc + j;  in ipu_gc_tasklet()
1317 			spin_lock_irqsave(&ichan->lock, flags);  in ipu_gc_tasklet()
1319 				list_move(&desc->list, &ichan->free_list);  in ipu_gc_tasklet()
1321 					if (ichan->sg[0] == sg)  in ipu_gc_tasklet()
1322 						ichan->sg[0] = NULL;  in ipu_gc_tasklet()
1323 					else if (ichan->sg[1] == sg)  in ipu_gc_tasklet()
1324 						ichan->sg[1] = NULL;  in ipu_gc_tasklet()
1328 			spin_unlock_irqrestore(&ichan->lock, flags);  in ipu_gc_tasklet()
1339 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_prep_slave_sg()  local
1354 	mutex_lock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1356 	spin_lock_irqsave(&ichan->lock, flags);  in idmac_prep_slave_sg()
1357 	if (!list_empty(&ichan->free_list)) {  in idmac_prep_slave_sg()
1358 		desc = list_entry(ichan->free_list.next,  in idmac_prep_slave_sg()
1368 	spin_unlock_irqrestore(&ichan->lock, flags);  in idmac_prep_slave_sg()
1370 	mutex_unlock(&ichan->chan_mutex);  in idmac_prep_slave_sg()
1380 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_issue_pending()  local
1387 	ipu_select_buffer(chan->chan_id, ichan->active_buffer);  in idmac_issue_pending()
1400 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_pause()  local
1406 	mutex_lock(&ichan->chan_mutex);  in idmac_pause()
1412 	list_for_each_safe(list, tmp, &ichan->queue)  in idmac_pause()
1415 	ichan->sg[0] = NULL;  in idmac_pause()
1416 	ichan->sg[1] = NULL;  in idmac_pause()
1420 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_pause()
1422 	mutex_unlock(&ichan->chan_mutex);  in idmac_pause()
1429 	struct idmac_channel *ichan = to_idmac_chan(chan);  in __idmac_terminate_all()  local
1435 	ipu_disable_channel(idmac, ichan,  in __idmac_terminate_all()
1436 			    ichan->status >= IPU_CHANNEL_ENABLED);  in __idmac_terminate_all()
1441 	spin_lock_irqsave(&ichan->lock, flags);  in __idmac_terminate_all()
1442 	list_splice_init(&ichan->queue, &ichan->free_list);  in __idmac_terminate_all()
1444 	if (ichan->desc)  in __idmac_terminate_all()
1445 		for (i = 0; i < ichan->n_tx_desc; i++) {  in __idmac_terminate_all()
1446 			struct idmac_tx_desc *desc = ichan->desc + i;  in __idmac_terminate_all()
1449 				list_add(&desc->list, &ichan->free_list);  in __idmac_terminate_all()
1454 	ichan->sg[0] = NULL;  in __idmac_terminate_all()
1455 	ichan->sg[1] = NULL;  in __idmac_terminate_all()
1456 	spin_unlock_irqrestore(&ichan->lock, flags);  in __idmac_terminate_all()
1460 	ichan->status = IPU_CHANNEL_INITIALIZED;  in __idmac_terminate_all()
1467 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_terminate_all()  local
1470 	mutex_lock(&ichan->chan_mutex);  in idmac_terminate_all()
1474 	mutex_unlock(&ichan->chan_mutex);  in idmac_terminate_all()
1482 	struct idmac_channel *ichan = dev_id;  in ic_sof_irq()  local
1484 	       irq, ichan->dma_chan.chan_id);  in ic_sof_irq()
1491 	struct idmac_channel *ichan = dev_id;  in ic_eof_irq()  local
1493 	       irq, ichan->dma_chan.chan_id);  in ic_eof_irq()
1503 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_alloc_chan_resources()  local
1509 	WARN_ON(ichan->status != IPU_CHANNEL_FREE);  in idmac_alloc_chan_resources()
1517 	ichan->eof_irq = ret;  in idmac_alloc_chan_resources()
1523 	ipu_disable_channel(idmac, ichan, true);  in idmac_alloc_chan_resources()
1525 	ret = ipu_init_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1529 	ret = request_irq(ichan->eof_irq, idmac_interrupt, 0,  in idmac_alloc_chan_resources()
1530 			  ichan->eof_name, ichan);  in idmac_alloc_chan_resources()
1538 			ret = request_irq(ic_sof, ic_sof_irq, 0, "IC SOF", ichan);  in idmac_alloc_chan_resources()
1544 			ret = request_irq(ic_eof, ic_eof_irq, 0, "IC EOF", ichan);  in idmac_alloc_chan_resources()
1551 	ichan->status = IPU_CHANNEL_INITIALIZED;  in idmac_alloc_chan_resources()
1554 		chan->chan_id, ichan->eof_irq);  in idmac_alloc_chan_resources()
1559 	ipu_uninit_channel(idmac, ichan);  in idmac_alloc_chan_resources()
1568 	struct idmac_channel *ichan = to_idmac_chan(chan);  in idmac_free_chan_resources()  local
1571 	mutex_lock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1575 	if (ichan->status > IPU_CHANNEL_FREE) {  in idmac_free_chan_resources()
1579 				free_irq(ic_sof, ichan);  in idmac_free_chan_resources()
1584 				free_irq(ic_eof, ichan);  in idmac_free_chan_resources()
1590 		free_irq(ichan->eof_irq, ichan);  in idmac_free_chan_resources()
1594 	ichan->status = IPU_CHANNEL_FREE;  in idmac_free_chan_resources()
1596 	ipu_uninit_channel(idmac, ichan);  in idmac_free_chan_resources()
1598 	mutex_unlock(&ichan->chan_mutex);  in idmac_free_chan_resources()
1632 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_init()  local
1633 		struct dma_chan *dma_chan = &ichan->dma_chan;  in ipu_idmac_init()
1635 		spin_lock_init(&ichan->lock);  in ipu_idmac_init()
1636 		mutex_init(&ichan->chan_mutex);  in ipu_idmac_init()
1638 		ichan->status		= IPU_CHANNEL_FREE;  in ipu_idmac_init()
1639 		ichan->sec_chan_en	= false;  in ipu_idmac_init()
1640 		snprintf(ichan->eof_name, sizeof(ichan->eof_name), "IDMAC EOF %d", i);  in ipu_idmac_init()
1659 		struct idmac_channel *ichan = ipu->channel + i;  in ipu_idmac_exit()  local
1661 		idmac_terminate_all(&ichan->dma_chan);  in ipu_idmac_exit()