41 static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum)  in read_txq_cur_addr()  argument
46 	txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));  in read_txq_cur_addr()
47 	txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));  in read_txq_cur_addr()
52 static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum)  in read_rxq_cur_addr()  argument
57 	rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));  in read_rxq_cur_addr()
58 	rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));  in read_rxq_cur_addr()
63 static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)  in write_txq_start_addr()  argument
67 	mtu3_writel(mbase, USB_QMU_TQSAR(epnum),  in write_txq_start_addr()
69 	tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));  in write_txq_start_addr()
72 	mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar);  in write_txq_start_addr()
75 static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)  in write_rxq_start_addr()  argument
79 	mtu3_writel(mbase, USB_QMU_RQSAR(epnum),  in write_rxq_start_addr()
81 	rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));  in write_rxq_start_addr()
84 	mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar);  in write_rxq_start_addr()
182 	int epnum = mep->epnum;  in mtu3_qmu_resume()  local
185 	offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_resume()
253 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_tx_gpd()
292 		mep->epnum, gpd, enq, &enq_dma);  in mtu3_prepare_rx_gpd()
320 	u8 epnum = mep->epnum;  in mtu3_qmu_start()  local
324 		write_txq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
325 		mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);  in mtu3_qmu_start()
326 		mtu3_setbits(mbase, U3D_QCR0, QMU_TX_CS_EN(epnum));  in mtu3_qmu_start()
328 		mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));  in mtu3_qmu_start()
330 				QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));  in mtu3_qmu_start()
332 		if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {  in mtu3_qmu_start()
333 			dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);  in mtu3_qmu_start()
336 		mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);  in mtu3_qmu_start()
339 		write_rxq_start_addr(mbase, epnum, ring->dma);  in mtu3_qmu_start()
340 		mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);  in mtu3_qmu_start()
341 		mtu3_setbits(mbase, U3D_QCR0, QMU_RX_CS_EN(epnum));  in mtu3_qmu_start()
343 		mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));  in mtu3_qmu_start()
345 		mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));  in mtu3_qmu_start()
347 				QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));  in mtu3_qmu_start()
348 		mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));  in mtu3_qmu_start()
350 		if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {  in mtu3_qmu_start()
351 			dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);  in mtu3_qmu_start()
354 		mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);  in mtu3_qmu_start()
365 	int epnum = mep->epnum;  in mtu3_qmu_stop()  local
370 	qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);  in mtu3_qmu_stop()
404 static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)  in qmu_tx_zlp_error_handler()  argument
406 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_tx_zlp_error_handler()
422 	cur_gpd_dma = read_txq_cur_addr(mbase, epnum);  in qmu_tx_zlp_error_handler()
426 		dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);  in qmu_tx_zlp_error_handler()
432 	mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
434 	ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),  in qmu_tx_zlp_error_handler()
440 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);  in qmu_tx_zlp_error_handler()
448 	mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);  in qmu_tx_zlp_error_handler()
459 static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)  in qmu_done_tx()  argument
461 	struct mtu3_ep *mep = mtu->in_eps + epnum;  in qmu_done_tx()
471 	cur_gpd_dma = read_txq_cur_addr(mbase, epnum);  in qmu_done_tx()
475 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_tx()
494 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_tx()
498 static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)  in qmu_done_rx()  argument
500 	struct mtu3_ep *mep = mtu->out_eps + epnum;  in qmu_done_rx()
509 	cur_gpd_dma = read_rxq_cur_addr(mbase, epnum);  in qmu_done_rx()
513 		__func__, epnum, gpd, gpd_current, ring->enqueue);  in qmu_done_rx()
532 		__func__, epnum, ring->dequeue, ring->enqueue);  in qmu_done_rx()