186 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_reset_chan()  local
187 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_reset_chan()
188 	int chan_id = mxs_chan->chan.chan_id;  in mxs_dma_reset_chan()
197 	if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE &&  in mxs_dma_reset_chan()
198 			mxs_chan->flags & MXS_DMA_SG_LOOP) {  in mxs_dma_reset_chan()
199 		mxs_chan->reset = true;  in mxs_dma_reset_chan()
222 			dev_err(&mxs_chan->mxs_dma->pdev->dev,  in mxs_dma_reset_chan()
230 	mxs_chan->status = DMA_COMPLETE;  in mxs_dma_reset_chan()
235 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_enable_chan()  local
236 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_enable_chan()
237 	int chan_id = mxs_chan->chan.chan_id;  in mxs_dma_enable_chan()
240 	writel(mxs_chan->ccw_phys,  in mxs_dma_enable_chan()
244 	if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE &&  in mxs_dma_enable_chan()
245 			mxs_chan->flags & MXS_DMA_SG_LOOP) {  in mxs_dma_enable_chan()
253 	mxs_chan->reset = false;  in mxs_dma_enable_chan()
258 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_disable_chan()  local
260 	mxs_chan->status = DMA_COMPLETE;  in mxs_dma_disable_chan()
265 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_pause_chan()  local
266 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_pause_chan()
267 	int chan_id = mxs_chan->chan.chan_id;  in mxs_dma_pause_chan()
277 	mxs_chan->status = DMA_PAUSED;  in mxs_dma_pause_chan()
283 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_resume_chan()  local
284 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_resume_chan()
285 	int chan_id = mxs_chan->chan.chan_id;  in mxs_dma_resume_chan()
295 	mxs_chan->status = DMA_IN_PROGRESS;  in mxs_dma_resume_chan()
306 	struct mxs_dma_chan *mxs_chan = from_tasklet(mxs_chan, t, tasklet);  in mxs_dma_tasklet()  local
308 	dmaengine_desc_get_callback_invoke(&mxs_chan->desc, NULL);  in mxs_dma_tasklet()
325 	struct mxs_dma_chan *mxs_chan;  in mxs_dma_int_handler()  local
365 	mxs_chan = &mxs_dma->mxs_chans[chan];  in mxs_dma_int_handler()
371 		mxs_chan->status = DMA_ERROR;  in mxs_dma_int_handler()
372 		mxs_dma_reset_chan(&mxs_chan->chan);  in mxs_dma_int_handler()
373 	} else if (mxs_chan->status != DMA_COMPLETE) {  in mxs_dma_int_handler()
374 		if (mxs_chan->flags & MXS_DMA_SG_LOOP) {  in mxs_dma_int_handler()
375 			mxs_chan->status = DMA_IN_PROGRESS;  in mxs_dma_int_handler()
376 			if (mxs_chan->flags & MXS_DMA_USE_SEMAPHORE)  in mxs_dma_int_handler()
380 			mxs_chan->status = DMA_COMPLETE;  in mxs_dma_int_handler()
384 	if (mxs_chan->status == DMA_COMPLETE) {  in mxs_dma_int_handler()
385 		if (mxs_chan->reset)  in mxs_dma_int_handler()
387 		dma_cookie_complete(&mxs_chan->desc);  in mxs_dma_int_handler()
391 	tasklet_schedule(&mxs_chan->tasklet);  in mxs_dma_int_handler()
398 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_alloc_chan_resources()  local
399 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_alloc_chan_resources()
402 	mxs_chan->ccw = dma_alloc_coherent(mxs_dma->dma_device.dev,  in mxs_dma_alloc_chan_resources()
404 					   &mxs_chan->ccw_phys, GFP_KERNEL);  in mxs_dma_alloc_chan_resources()
405 	if (!mxs_chan->ccw) {  in mxs_dma_alloc_chan_resources()
410 	ret = request_irq(mxs_chan->chan_irq, mxs_dma_int_handler,  in mxs_dma_alloc_chan_resources()
421 	dma_async_tx_descriptor_init(&mxs_chan->desc, chan);  in mxs_dma_alloc_chan_resources()
422 	mxs_chan->desc.tx_submit = mxs_dma_tx_submit;  in mxs_dma_alloc_chan_resources()
425 	async_tx_ack(&mxs_chan->desc);  in mxs_dma_alloc_chan_resources()
430 	free_irq(mxs_chan->chan_irq, mxs_dma);  in mxs_dma_alloc_chan_resources()
433 			mxs_chan->ccw, mxs_chan->ccw_phys);  in mxs_dma_alloc_chan_resources()
440 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_free_chan_resources()  local
441 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_free_chan_resources()
445 	free_irq(mxs_chan->chan_irq, mxs_dma);  in mxs_dma_free_chan_resources()
448 			mxs_chan->ccw, mxs_chan->ccw_phys);  in mxs_dma_free_chan_resources()
480 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_prep_slave_sg()  local
481 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_prep_slave_sg()
488 	if (mxs_chan->status == DMA_IN_PROGRESS)  in mxs_dma_prep_slave_sg()
489 		idx = mxs_chan->desc_count;  in mxs_dma_prep_slave_sg()
498 	mxs_chan->status = DMA_IN_PROGRESS;  in mxs_dma_prep_slave_sg()
499 	mxs_chan->flags = 0;  in mxs_dma_prep_slave_sg()
507 		ccw = &mxs_chan->ccw[idx - 1];  in mxs_dma_prep_slave_sg()
508 		ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;  in mxs_dma_prep_slave_sg()
517 		ccw = &mxs_chan->ccw[idx++];  in mxs_dma_prep_slave_sg()
542 			ccw = &mxs_chan->ccw[idx++];  in mxs_dma_prep_slave_sg()
544 			ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * idx;  in mxs_dma_prep_slave_sg()
565 	mxs_chan->desc_count = idx;  in mxs_dma_prep_slave_sg()
567 	return &mxs_chan->desc;  in mxs_dma_prep_slave_sg()
570 	mxs_chan->status = DMA_ERROR;  in mxs_dma_prep_slave_sg()
579 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_prep_dma_cyclic()  local
580 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_prep_dma_cyclic()
584 	if (mxs_chan->status == DMA_IN_PROGRESS)  in mxs_dma_prep_dma_cyclic()
587 	mxs_chan->status = DMA_IN_PROGRESS;  in mxs_dma_prep_dma_cyclic()
588 	mxs_chan->flags |= MXS_DMA_SG_LOOP;  in mxs_dma_prep_dma_cyclic()
589 	mxs_chan->flags |= MXS_DMA_USE_SEMAPHORE;  in mxs_dma_prep_dma_cyclic()
606 		struct mxs_dma_ccw *ccw = &mxs_chan->ccw[i];  in mxs_dma_prep_dma_cyclic()
609 			ccw->next = mxs_chan->ccw_phys;  in mxs_dma_prep_dma_cyclic()
611 			ccw->next = mxs_chan->ccw_phys + sizeof(*ccw) * (i + 1);  in mxs_dma_prep_dma_cyclic()
630 	mxs_chan->desc_count = i;  in mxs_dma_prep_dma_cyclic()
632 	return &mxs_chan->desc;  in mxs_dma_prep_dma_cyclic()
635 	mxs_chan->status = DMA_ERROR;  in mxs_dma_prep_dma_cyclic()
650 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_tx_status()  local
651 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_tx_status()
654 	if (mxs_chan->status == DMA_IN_PROGRESS &&  in mxs_dma_tx_status()
655 			mxs_chan->flags & MXS_DMA_SG_LOOP) {  in mxs_dma_tx_status()
659 		last_ccw = &mxs_chan->ccw[mxs_chan->desc_count - 1];  in mxs_dma_tx_status()
670 	return mxs_chan->status;  in mxs_dma_tx_status()
709 	struct mxs_dma_chan *mxs_chan = to_mxs_dma_chan(chan);  in mxs_dma_filter_fn()  local
710 	struct mxs_dma_engine *mxs_dma = mxs_chan->mxs_dma;  in mxs_dma_filter_fn()
720 	mxs_chan->chan_irq = chan_irq;  in mxs_dma_filter_fn()
782 		struct mxs_dma_chan *mxs_chan = &mxs_dma->mxs_chans[i];  in mxs_dma_probe()  local
784 		mxs_chan->mxs_dma = mxs_dma;  in mxs_dma_probe()
785 		mxs_chan->chan.device = &mxs_dma->dma_device;  in mxs_dma_probe()
786 		dma_cookie_init(&mxs_chan->chan);  in mxs_dma_probe()
788 		tasklet_setup(&mxs_chan->tasklet, mxs_dma_tasklet);  in mxs_dma_probe()
792 		list_add_tail(&mxs_chan->chan.device_node,  in mxs_dma_probe()