74 	struct ccp_dma_desc *desc, *dtmp;  in ccp_free_desc_resources()  local
76 	list_for_each_entry_safe(desc, dtmp, list, entry) {  in ccp_free_desc_resources()
77 		ccp_free_cmd_resources(ccp, &desc->active);  in ccp_free_desc_resources()
78 		ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_free_desc_resources()
80 		list_del(&desc->entry);  in ccp_free_desc_resources()
81 		kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_free_desc_resources()
106 	struct ccp_dma_desc *desc, *dtmp;  in ccp_cleanup_desc_resources()  local
108 	list_for_each_entry_safe_reverse(desc, dtmp, list, entry) {  in ccp_cleanup_desc_resources()
109 		if (!async_tx_test_ack(&desc->tx_desc))  in ccp_cleanup_desc_resources()
112 		dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);  in ccp_cleanup_desc_resources()
114 		ccp_free_cmd_resources(ccp, &desc->active);  in ccp_cleanup_desc_resources()
115 		ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_cleanup_desc_resources()
117 		list_del(&desc->entry);  in ccp_cleanup_desc_resources()
118 		kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_cleanup_desc_resources()
137 static int ccp_issue_next_cmd(struct ccp_dma_desc *desc)  in ccp_issue_next_cmd()  argument
142 	cmd = list_first_entry(&desc->pending, struct ccp_dma_cmd, entry);  in ccp_issue_next_cmd()
143 	list_move(&cmd->entry, &desc->active);  in ccp_issue_next_cmd()
145 	dev_dbg(desc->ccp->dev, "%s - tx %d, cmd=%p\n", __func__,  in ccp_issue_next_cmd()
146 		desc->tx_desc.cookie, cmd);  in ccp_issue_next_cmd()
152 	dev_dbg(desc->ccp->dev, "%s - error: ret=%d, tx %d, cmd=%p\n", __func__,  in ccp_issue_next_cmd()
153 		ret, desc->tx_desc.cookie, cmd);  in ccp_issue_next_cmd()
158 static void ccp_free_active_cmd(struct ccp_dma_desc *desc)  in ccp_free_active_cmd()  argument
162 	cmd = list_first_entry_or_null(&desc->active, struct ccp_dma_cmd,  in ccp_free_active_cmd()
167 	dev_dbg(desc->ccp->dev, "%s - freeing tx %d cmd=%p\n",  in ccp_free_active_cmd()
168 		__func__, desc->tx_desc.cookie, cmd);  in ccp_free_active_cmd()
171 	kmem_cache_free(desc->ccp->dma_cmd_cache, cmd);  in ccp_free_active_cmd()
175 						struct ccp_dma_desc *desc)  in __ccp_next_dma_desc()  argument
178 	if (desc)  in __ccp_next_dma_desc()
179 		list_move(&desc->entry, &chan->complete);  in __ccp_next_dma_desc()
182 	desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,  in __ccp_next_dma_desc()
185 	return desc;  in __ccp_next_dma_desc()
189 						   struct ccp_dma_desc *desc)  in ccp_handle_active_desc()  argument
196 		if (desc) {  in ccp_handle_active_desc()
198 			ccp_free_active_cmd(desc);  in ccp_handle_active_desc()
200 			if (!list_empty(&desc->pending)) {  in ccp_handle_active_desc()
202 				if (desc->status != DMA_ERROR)  in ccp_handle_active_desc()
203 					return desc;  in ccp_handle_active_desc()
206 				ccp_free_cmd_resources(desc->ccp,  in ccp_handle_active_desc()
207 						       &desc->pending);  in ccp_handle_active_desc()
210 			tx_desc = &desc->tx_desc;  in ccp_handle_active_desc()
217 		if (desc) {  in ccp_handle_active_desc()
218 			if (desc->status != DMA_ERROR)  in ccp_handle_active_desc()
219 				desc->status = DMA_COMPLETE;  in ccp_handle_active_desc()
221 			dev_dbg(desc->ccp->dev,  in ccp_handle_active_desc()
223 				desc->tx_desc.cookie, desc->status);  in ccp_handle_active_desc()
229 		desc = __ccp_next_dma_desc(chan, desc);  in ccp_handle_active_desc()
238 	} while (desc);  in ccp_handle_active_desc()
245 	struct ccp_dma_desc *desc;  in __ccp_pending_to_active()  local
250 	desc = list_empty(&chan->active)  in __ccp_pending_to_active()
256 	return desc;  in __ccp_pending_to_active()
261 	struct ccp_dma_desc *desc = data;  in ccp_cmd_callback()  local
268 	chan = container_of(desc->tx_desc.chan, struct ccp_dma_chan,  in ccp_cmd_callback()
272 		__func__, desc->tx_desc.cookie, err);  in ccp_cmd_callback()
275 		desc->status = DMA_ERROR;  in ccp_cmd_callback()
279 		desc = ccp_handle_active_desc(chan, desc);  in ccp_cmd_callback()
282 		if (!desc || (chan->status == DMA_PAUSED))  in ccp_cmd_callback()
285 		ret = ccp_issue_next_cmd(desc);  in ccp_cmd_callback()
289 		desc->status = DMA_ERROR;  in ccp_cmd_callback()
297 	struct ccp_dma_desc *desc = container_of(tx_desc, struct ccp_dma_desc,  in ccp_tx_submit()  local
308 	list_del(&desc->entry);  in ccp_tx_submit()
309 	list_add_tail(&desc->entry, &chan->pending);  in ccp_tx_submit()
333 	struct ccp_dma_desc *desc;  in ccp_alloc_dma_desc()  local
335 	desc = kmem_cache_zalloc(chan->ccp->dma_desc_cache, GFP_NOWAIT);  in ccp_alloc_dma_desc()
336 	if (!desc)  in ccp_alloc_dma_desc()
339 	dma_async_tx_descriptor_init(&desc->tx_desc, &chan->dma_chan);  in ccp_alloc_dma_desc()
340 	desc->tx_desc.flags = flags;  in ccp_alloc_dma_desc()
341 	desc->tx_desc.tx_submit = ccp_tx_submit;  in ccp_alloc_dma_desc()
342 	desc->ccp = chan->ccp;  in ccp_alloc_dma_desc()
343 	INIT_LIST_HEAD(&desc->pending);  in ccp_alloc_dma_desc()
344 	INIT_LIST_HEAD(&desc->active);  in ccp_alloc_dma_desc()
345 	desc->status = DMA_IN_PROGRESS;  in ccp_alloc_dma_desc()
347 	return desc;  in ccp_alloc_dma_desc()
360 	struct ccp_dma_desc *desc;  in ccp_create_desc()  local
376 	desc = ccp_alloc_dma_desc(chan, flags);  in ccp_create_desc()
377 	if (!desc)  in ccp_create_desc()
436 		ccp_cmd->data = desc;  in ccp_create_desc()
438 		list_add_tail(&cmd->entry, &desc->pending);  in ccp_create_desc()
454 	desc->len = total_len;  in ccp_create_desc()
456 	if (list_empty(&desc->pending))  in ccp_create_desc()
459 	dev_dbg(ccp->dev, "%s - desc=%p\n", __func__, desc);  in ccp_create_desc()
463 	list_add_tail(&desc->entry, &chan->created);  in ccp_create_desc()
467 	return desc;  in ccp_create_desc()
470 	ccp_free_cmd_resources(ccp, &desc->pending);  in ccp_create_desc()
471 	kmem_cache_free(ccp->dma_desc_cache, desc);  in ccp_create_desc()
482 	struct ccp_dma_desc *desc;  in ccp_prep_dma_memcpy()  local
497 	desc = ccp_create_desc(dma_chan, &dst_sg, 1, &src_sg, 1, flags);  in ccp_prep_dma_memcpy()
498 	if (!desc)  in ccp_prep_dma_memcpy()
501 	return &desc->tx_desc;  in ccp_prep_dma_memcpy()
509 	struct ccp_dma_desc *desc;  in ccp_prep_dma_interrupt()  local
511 	desc = ccp_alloc_dma_desc(chan, flags);  in ccp_prep_dma_interrupt()
512 	if (!desc)  in ccp_prep_dma_interrupt()
515 	return &desc->tx_desc;  in ccp_prep_dma_interrupt()
522 	struct ccp_dma_desc *desc;  in ccp_issue_pending()  local
529 	desc = __ccp_pending_to_active(chan);  in ccp_issue_pending()
534 	if (desc)  in ccp_issue_pending()
535 		ccp_cmd_callback(desc, 0);  in ccp_issue_pending()
544 	struct ccp_dma_desc *desc;  in ccp_tx_status()  local
558 		list_for_each_entry(desc, &chan->complete, entry) {  in ccp_tx_status()
559 			if (desc->tx_desc.cookie != cookie)  in ccp_tx_status()
562 			ret = desc->status;  in ccp_tx_status()
591 	struct ccp_dma_desc *desc;  in ccp_resume()  local
596 	desc = list_first_entry_or_null(&chan->active, struct ccp_dma_desc,  in ccp_resume()
605 	if (desc)  in ccp_resume()
606 		ccp_cmd_callback(desc, 0);  in ccp_resume()