154 static void reset_gpd_list(struct mtu3_ep *mep)  in reset_gpd_list()  argument
156 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in reset_gpd_list()
165 int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)  in mtu3_gpd_ring_alloc()  argument
168 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_alloc()
171 	gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);  in mtu3_gpd_ring_alloc()
180 void mtu3_gpd_ring_free(struct mtu3_ep *mep)  in mtu3_gpd_ring_free()  argument
182 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_gpd_ring_free()
184 	dma_pool_free(mep->mtu->qmu_gpd_pool,  in mtu3_gpd_ring_free()
189 void mtu3_qmu_resume(struct mtu3_ep *mep)  in mtu3_qmu_resume()  argument
191 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_resume()
193 	int epnum = mep->epnum;  in mtu3_qmu_resume()
196 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_resume()
238 int mtu3_prepare_transfer(struct mtu3_ep *mep)  in mtu3_prepare_transfer()  argument
240 	return gpd_ring_empty(&mep->gpd_ring);  in mtu3_prepare_transfer()
243 static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_prepare_tx_gpd()  argument
246 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_tx_gpd()
249 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_tx_gpd()
261 	dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_tx_gpd()
262 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_tx_gpd()
279 	trace_mtu3_prepare_gpd(mep, gpd);  in mtu3_prepare_tx_gpd()
284 static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_prepare_rx_gpd()  argument
287 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_prepare_rx_gpd()
290 	struct mtu3 *mtu = mep->mtu;  in mtu3_prepare_rx_gpd()
302 	dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",  in mtu3_prepare_rx_gpd()
303 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_rx_gpd()
312 	trace_mtu3_prepare_gpd(mep, gpd);  in mtu3_prepare_rx_gpd()
317 void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)  in mtu3_insert_gpd()  argument
320 	if (mep->is_in)  in mtu3_insert_gpd()
321 		mtu3_prepare_tx_gpd(mep, mreq);  in mtu3_insert_gpd()
323 		mtu3_prepare_rx_gpd(mep, mreq);  in mtu3_insert_gpd()
326 int mtu3_qmu_start(struct mtu3_ep *mep)  in mtu3_qmu_start()  argument
328 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_start()
330 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in mtu3_qmu_start()
331 	u8 epnum = mep->epnum;  in mtu3_qmu_start()
333 	if (mep->is_in) {  in mtu3_qmu_start()
370 void mtu3_qmu_stop(struct mtu3_ep *mep)  in mtu3_qmu_stop()  argument
372 	struct mtu3 *mtu = mep->mtu;  in mtu3_qmu_stop()
374 	int epnum = mep->epnum;  in mtu3_qmu_stop()
379 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_stop()
382 		dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);  in mtu3_qmu_stop()
390 		dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);  in mtu3_qmu_stop()
394 	dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);  in mtu3_qmu_stop()
397 void mtu3_qmu_flush(struct mtu3_ep *mep)  in mtu3_qmu_flush()  argument
400 	dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,  in mtu3_qmu_flush()
401 		((mep->is_in) ? "TX" : "RX"));  in mtu3_qmu_flush()
404 	mtu3_qmu_stop(mep);  in mtu3_qmu_flush()
405 	reset_gpd_list(mep);  in mtu3_qmu_flush()
415 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()  local
416 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_tx_zlp_error_handler()
424 	mreq = next_request(mep);  in qmu_tx_zlp_error_handler()
437 	trace_mtu3_zlp_exp_gpd(mep, gpd_current);  in qmu_tx_zlp_error_handler()
439 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
441 	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),  in qmu_tx_zlp_error_handler()
447 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);  in qmu_tx_zlp_error_handler()
453 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
454 	mtu3_qmu_resume(mep);  in qmu_tx_zlp_error_handler()
466 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()  local
467 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_tx()
484 		mreq = next_request(mep);  in qmu_done_tx()
493 		trace_mtu3_complete_gpd(mep, gpd);  in qmu_done_tx()
494 		mtu3_req_complete(mep, request, 0);  in qmu_done_tx()
506 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()  local
507 	struct mtu3_gpd_ring *ring = &mep->gpd_ring;  in qmu_done_rx()
523 		mreq = next_request(mep);  in qmu_done_rx()
532 		trace_mtu3_complete_gpd(mep, gpd);  in qmu_done_rx()
533 		mtu3_req_complete(mep, req, 0);  in qmu_done_rx()