108 static void hidma_process_completed(struct hidma_chan *mchan)  in hidma_process_completed()  argument
110 	struct dma_device *ddev = mchan->chan.device;  in hidma_process_completed()
122 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_process_completed()
123 	list_splice_tail_init(&mchan->completed, &list);  in hidma_process_completed()
124 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_process_completed()
137 		spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_process_completed()
139 			mchan->last_success = last_cookie;  in hidma_process_completed()
146 		spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_process_completed()
152 		spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_process_completed()
153 		list_move(&mdesc->node, &mchan->free);  in hidma_process_completed()
154 		spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_process_completed()
168 	struct hidma_chan *mchan = to_hidma_chan(mdesc->desc.chan);  in hidma_callback()  local
169 	struct dma_device *ddev = mchan->chan.device;  in hidma_callback()
174 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_callback()
177 		list_move_tail(&mdesc->node, &mchan->completed);  in hidma_callback()
181 		mchan->running = list_first_entry(&mchan->active,  in hidma_callback()
184 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_callback()
186 	hidma_process_completed(mchan);  in hidma_callback()
196 	struct hidma_chan *mchan;  in hidma_chan_init()  local
199 	mchan = devm_kzalloc(dmadev->ddev.dev, sizeof(*mchan), GFP_KERNEL);  in hidma_chan_init()
200 	if (!mchan)  in hidma_chan_init()
204 	mchan->dma_sig = dma_sig;  in hidma_chan_init()
205 	mchan->dmadev = dmadev;  in hidma_chan_init()
206 	mchan->chan.device = ddev;  in hidma_chan_init()
207 	dma_cookie_init(&mchan->chan);  in hidma_chan_init()
209 	INIT_LIST_HEAD(&mchan->free);  in hidma_chan_init()
210 	INIT_LIST_HEAD(&mchan->prepared);  in hidma_chan_init()
211 	INIT_LIST_HEAD(&mchan->active);  in hidma_chan_init()
212 	INIT_LIST_HEAD(&mchan->completed);  in hidma_chan_init()
213 	INIT_LIST_HEAD(&mchan->queued);  in hidma_chan_init()
215 	spin_lock_init(&mchan->lock);  in hidma_chan_init()
216 	list_add_tail(&mchan->chan.device_node, &ddev->channels);  in hidma_chan_init()
230 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_issue_pending()  local
231 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_issue_pending()
236 	spin_lock_irqsave(&mchan->lock, flags);  in hidma_issue_pending()
237 	list_for_each_entry_safe(qdesc, next, &mchan->queued, node) {  in hidma_issue_pending()
239 		list_move_tail(&qdesc->node, &mchan->active);  in hidma_issue_pending()
242 	if (!mchan->running) {  in hidma_issue_pending()
243 		struct hidma_desc *desc = list_first_entry(&mchan->active,  in hidma_issue_pending()
246 		mchan->running = desc;  in hidma_issue_pending()
248 	spin_unlock_irqrestore(&mchan->lock, flags);  in hidma_issue_pending()
275 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_tx_status()  local
282 		is_success = hidma_txn_is_success(cookie, mchan->last_success,  in hidma_tx_status()
287 	if (mchan->paused && (ret == DMA_IN_PROGRESS)) {  in hidma_tx_status()
291 		spin_lock_irqsave(&mchan->lock, flags);  in hidma_tx_status()
292 		if (mchan->running)  in hidma_tx_status()
293 			runcookie = mchan->running->desc.cookie;  in hidma_tx_status()
300 		spin_unlock_irqrestore(&mchan->lock, flags);  in hidma_tx_status()
312 	struct hidma_chan *mchan = to_hidma_chan(txd->chan);  in hidma_tx_submit()  local
313 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_tx_submit()
328 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_tx_submit()
331 	list_move_tail(&mdesc->node, &mchan->queued);  in hidma_tx_submit()
336 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_tx_submit()
343 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_alloc_chan_resources()  local
344 	struct hidma_dev *dmadev = mchan->dmadev;  in hidma_alloc_chan_resources()
351 	if (mchan->allocated)  in hidma_alloc_chan_resources()
364 		rc = hidma_ll_request(dmadev->lldev, mchan->dma_sig,  in hidma_alloc_chan_resources()
385 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_alloc_chan_resources()
386 	list_splice_tail_init(&descs, &mchan->free);  in hidma_alloc_chan_resources()
387 	mchan->allocated = true;  in hidma_alloc_chan_resources()
388 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_alloc_chan_resources()
396 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_prep_dma_memcpy()  local
398 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memcpy()
402 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_prep_dma_memcpy()
403 	if (!list_empty(&mchan->free)) {  in hidma_prep_dma_memcpy()
404 		mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);  in hidma_prep_dma_memcpy()
407 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_prep_dma_memcpy()
418 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_prep_dma_memcpy()
419 	list_add_tail(&mdesc->node, &mchan->prepared);  in hidma_prep_dma_memcpy()
420 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_prep_dma_memcpy()
429 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_prep_dma_memset()  local
431 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_prep_dma_memset()
436 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_prep_dma_memset()
437 	if (!list_empty(&mchan->free)) {  in hidma_prep_dma_memset()
438 		mdesc = list_first_entry(&mchan->free, struct hidma_desc, node);  in hidma_prep_dma_memset()
441 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_prep_dma_memset()
462 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_prep_dma_memset()
463 	list_add_tail(&mdesc->node, &mchan->prepared);  in hidma_prep_dma_memset()
464 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_prep_dma_memset()
471 	struct hidma_chan *mchan = to_hidma_chan(chan);  in hidma_terminate_channel()  local
472 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_channel()
480 	hidma_process_completed(mchan);  in hidma_terminate_channel()
482 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_terminate_channel()
483 	mchan->last_success = 0;  in hidma_terminate_channel()
484 	list_splice_init(&mchan->active, &list);  in hidma_terminate_channel()
485 	list_splice_init(&mchan->prepared, &list);  in hidma_terminate_channel()
486 	list_splice_init(&mchan->completed, &list);  in hidma_terminate_channel()
487 	list_splice_init(&mchan->queued, &list);  in hidma_terminate_channel()
488 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_terminate_channel()
506 		list_move(&mdesc->node, &mchan->free);  in hidma_terminate_channel()
518 	struct hidma_chan *mchan = to_hidma_chan(chan);  in hidma_terminate_all()  local
519 	struct hidma_dev *dmadev = to_hidma_dev(mchan->chan.device);  in hidma_terminate_all()
536 	struct hidma_chan *mchan = to_hidma_chan(dmach);  in hidma_free_chan_resources()  local
537 	struct hidma_dev *mdma = mchan->dmadev;  in hidma_free_chan_resources()
545 	spin_lock_irqsave(&mchan->lock, irqflags);  in hidma_free_chan_resources()
548 	list_splice_tail_init(&mchan->free, &descs);  in hidma_free_chan_resources()
557 	mchan->allocated = false;  in hidma_free_chan_resources()
558 	spin_unlock_irqrestore(&mchan->lock, irqflags);  in hidma_free_chan_resources()
563 	struct hidma_chan *mchan;  in hidma_pause()  local
566 	mchan = to_hidma_chan(chan);  in hidma_pause()
567 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_pause()
568 	if (!mchan->paused) {  in hidma_pause()
572 		mchan->paused = true;  in hidma_pause()
581 	struct hidma_chan *mchan;  in hidma_resume()  local
585 	mchan = to_hidma_chan(chan);  in hidma_resume()
586 	dmadev = to_hidma_dev(mchan->chan.device);  in hidma_resume()
587 	if (mchan->paused) {  in hidma_resume()
591 			mchan->paused = false;  in hidma_resume()