122 static void reset_gpd_list(struct mtu3_ep *mep)  in reset_gpd_list()  argument
124 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in reset_gpd_list()
133 int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)  in mtu3_gpd_ring_alloc()  argument
136 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_alloc()
139 	gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);  in mtu3_gpd_ring_alloc()
148 void mtu3_gpd_ring_free(struct mtu3_ep *mep)  in mtu3_gpd_ring_free()  argument
150 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_free()
152 	dma_pool_free(mep->mtu->qmu_gpd_pool,  in mtu3_gpd_ring_free()
178 void mtu3_qmu_resume(struct mtu3_ep *mep)  in mtu3_qmu_resume()  argument
180 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_resume()
182 	int epnum = mep->epnum;  in mtu3_qmu_resume()
185 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_resume()
227 int mtu3_prepare_transfer(struct mtu3_ep *mep)  in mtu3_prepare_transfer()  argument
229 	return gpd_ring_empty(&mep->gpd_ring);  in mtu3_prepare_transfer()
232 static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_prepare_tx_gpd()  argument
235 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_tx_gpd()
252 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_tx_gpd()
253 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_tx_gpd()
271 static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_prepare_rx_gpd()  argument
274 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_rx_gpd()
291 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_rx_gpd()
292 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_rx_gpd()
306 void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_insert_gpd()  argument
309 	if (mep->is_in)  in mtu3_insert_gpd()
310 		mtu3_prepare_tx_gpd(mep, mreq);  in mtu3_insert_gpd()
312 		mtu3_prepare_rx_gpd(mep, mreq);  in mtu3_insert_gpd()
315 int mtu3_qmu_start(struct mtu3_ep *mep)  in mtu3_qmu_start()  argument
317 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_start()
319 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_qmu_start()
320 	u8 epnum = mep->epnum;  in mtu3_qmu_start()
322 	if (mep->is_in) {  in mtu3_qmu_start()
361 void mtu3_qmu_stop(struct mtu3_ep *mep)  in mtu3_qmu_stop()  argument
363 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_stop()
365 	int epnum = mep->epnum;  in mtu3_qmu_stop()
370 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_stop()
373 		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);  in mtu3_qmu_stop()
381 		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);  in mtu3_qmu_stop()
385 	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);  in mtu3_qmu_stop()
388 void mtu3_qmu_flush(struct mtu3_ep *mep)  in mtu3_qmu_flush()  argument
391 	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,  in mtu3_qmu_flush()
392 		((mep->is_in) ? "TX" : "RX"));  in mtu3_qmu_flush()
395 	mtu3_qmu_stop(mep);  in mtu3_qmu_flush()
396 	reset_gpd_list(mep);  in mtu3_qmu_flush()
406 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()  local
407 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_tx_zlp_error_handler()
416 	mreq = next_request(mep);  in qmu_tx_zlp_error_handler()
432 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
434 	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),  in qmu_tx_zlp_error_handler()
440 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);  in qmu_tx_zlp_error_handler()
448 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
449 	mtu3_qmu_resume(mep);  in qmu_tx_zlp_error_handler()
461 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()  local
462 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_tx()
479 		mreq = next_request(mep);  in qmu_done_tx()
488 		mtu3_req_complete(mep, request, 0);  in qmu_done_tx()
500 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()  local
501 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_rx()
517 		mreq = next_request(mep);  in qmu_done_rx()
526 		mtu3_req_complete(mep, req, 0);  in qmu_done_rx()