98 				 struct usbhs_fifo *fifo);
99 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,
109 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhs_pkt_pop()  local
123 		if (fifo)  in usbhs_pkt_pop()
124 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
141 	if (fifo)  in usbhs_pkt_pop()
142 		usbhsf_fifo_unselect(pipe, fifo);  in usbhs_pkt_pop()
255  *		FIFO ctrl
258 				   struct usbhs_fifo *fifo)  in usbhsf_send_terminator()  argument
262 	usbhs_bset(priv, fifo->ctr, BVAL, BVAL);  in usbhsf_send_terminator()
266 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_barrier()  argument
268 	/* The FIFO port is accessible */  in usbhsf_fifo_barrier()
269 	if (usbhs_read(priv, fifo->ctr) & FRDY)  in usbhsf_fifo_barrier()
276 			      struct usbhs_fifo *fifo)  in usbhsf_fifo_clear()  argument
290 			ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_fifo_clear()
298 		usbhs_write(priv, fifo->ctr, BCLR);  in usbhsf_fifo_clear()
302 			       struct usbhs_fifo *fifo)  in usbhsf_fifo_rcv_len()  argument
304 	return usbhs_read(priv, fifo->ctr) & DTLN_MASK;  in usbhsf_fifo_rcv_len()
308 				 struct usbhs_fifo *fifo)  in usbhsf_fifo_unselect()  argument
313 	usbhs_write(priv, fifo->sel, 0);  in usbhsf_fifo_unselect()
317 			      struct usbhs_fifo *fifo,  in usbhsf_fifo_select()  argument
327 	    usbhsf_fifo_is_busy(fifo))  in usbhsf_fifo_select()
338 	usbhs_write(priv, fifo->sel, base | MBW_32);  in usbhsf_fifo_select()
342 		if (base == (mask & usbhs_read(priv, fifo->sel))) {  in usbhsf_fifo_select()
343 			usbhs_pipe_select_fifo(pipe, fifo);  in usbhsf_fifo_select()
349 	dev_err(dev, "fifo select error\n");  in usbhsf_fifo_select()
361 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_write()  local
367 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhs_dcp_dir_switch_to_write()
375 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
376 	usbhsf_send_terminator(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
378 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
390 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_dcp_dir_switch_to_read()  local
396 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhs_dcp_dir_switch_to_read()
403 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
405 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
468 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_dcp_data_stage_prepare_pop()  local
476 	 *  - clear fifo  in usbhsf_dcp_data_stage_prepare_pop()
483 	usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dcp_data_stage_prepare_pop()
484 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
485 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
507 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_push()  local
508 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_push()
520 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhsf_pio_try_push()
531 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_push()
577 		usbhsf_send_terminator(pipe, fifo);  in usbhsf_pio_try_push()
587 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
592 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
624 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv);  in usbhsf_prepare_pop()  local
639 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_prepare_pop()
654 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhsf_pio_try_pop()  local
655 	void __iomem *addr = priv->base + fifo->port;  in usbhsf_pio_try_pop()
663 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_pio_try_pop()
667 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_pio_try_pop()
671 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_pio_try_pop()
704 	 * "Operation" - "FIFO Buffer Memory" - "FIFO Port Function"  in usbhsf_pio_try_pop()
708 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_pio_try_pop()
737 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_pop()
765  *		DMA fifo functions
767 static struct dma_chan *usbhsf_dma_chan_get(struct usbhs_fifo *fifo,  in usbhsf_dma_chan_get()  argument
771 		return fifo->tx_chan;  in usbhsf_dma_chan_get()
774 		return fifo->rx_chan;  in usbhsf_dma_chan_get()
782 	struct usbhs_fifo *fifo;  in usbhsf_get_dma_fifo()  local
785 	usbhs_for_each_dfifo(priv, fifo, i) {  in usbhsf_get_dma_fifo()
786 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
787 		    !usbhsf_fifo_is_busy(fifo))  in usbhsf_get_dma_fifo()
788 			return fifo;  in usbhsf_get_dma_fifo()
797 			      struct usbhs_fifo *fifo,  in __usbhsf_dma_ctrl()  argument
802 	usbhs_bset(priv, fifo->sel, DREQE, dreqe);  in __usbhsf_dma_ctrl()
810 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in __usbhsf_dma_map_ctrl()  local
811 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in __usbhsf_dma_map_ctrl()
821 	struct usbhs_fifo *fifo;  in usbhsf_dma_xfer_preparing()  local
829 	fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_xfer_preparing()
830 	if (!fifo)  in usbhsf_dma_xfer_preparing()
833 	chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_xfer_preparing()
852 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in usbhsf_dma_xfer_preparing()
857 	usbhsf_dma_start(pipe, fifo);  in usbhsf_dma_xfer_preparing()
880 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_push()  local
907 	/* get enable DMA fifo */  in usbhsf_dma_prepare_push()
908 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
909 	if (!fifo)  in usbhsf_dma_prepare_push()
912 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_push()
933 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_push()
959 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_push_done()
961 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_push_done()
992 	struct usbhs_fifo *fifo;  in usbhsf_dma_prepare_pop_with_usb_dmac()  local
1003 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1004 	if (!fifo)  in usbhsf_dma_prepare_pop_with_usb_dmac()
1016 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1039 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1065 	struct usbhs_fifo *fifo;  in usbhsf_dma_try_pop_with_rx_irq()  local
1074 	/* get enable DMA fifo */  in usbhsf_dma_try_pop_with_rx_irq()
1075 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1076 	if (!fifo)  in usbhsf_dma_try_pop_with_rx_irq()
1082 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1087 	len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1095 	ret = usbhsf_fifo_barrier(priv, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1119 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1144 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1146 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1186 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()  local
1187 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1197 	rcv_len = usbhsf_fifo_rcv_len(priv, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1198 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1202 	usbhsf_dma_stop(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1204 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1248 static void usbhsf_dma_quit(struct usbhs_priv *priv, struct usbhs_fifo *fifo)  in usbhsf_dma_quit()  argument
1250 	if (fifo->tx_chan)  in usbhsf_dma_quit()
1251 		dma_release_channel(fifo->tx_chan);  in usbhsf_dma_quit()
1252 	if (fifo->rx_chan)  in usbhsf_dma_quit()
1253 		dma_release_channel(fifo->rx_chan);  in usbhsf_dma_quit()
1255 	fifo->tx_chan = NULL;  in usbhsf_dma_quit()
1256 	fifo->rx_chan = NULL;  in usbhsf_dma_quit()
1259 static void usbhsf_dma_init_pdev(struct usbhs_fifo *fifo)  in usbhsf_dma_init_pdev()  argument
1265 	fifo->tx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1266 					    &fifo->tx_slave);  in usbhsf_dma_init_pdev()
1270 	fifo->rx_chan = dma_request_channel(mask, usbhsf_dma_filter,  in usbhsf_dma_init_pdev()
1271 					    &fifo->rx_slave);  in usbhsf_dma_init_pdev()
1274 static void usbhsf_dma_init_dt(struct device *dev, struct usbhs_fifo *fifo,  in usbhsf_dma_init_dt()  argument
1286 		fifo->tx_chan = dma_request_chan(dev, name);  in usbhsf_dma_init_dt()
1287 		if (IS_ERR(fifo->tx_chan))  in usbhsf_dma_init_dt()
1288 			fifo->tx_chan = NULL;  in usbhsf_dma_init_dt()
1290 		fifo->rx_chan = dma_request_chan(dev, name);  in usbhsf_dma_init_dt()
1291 		if (IS_ERR(fifo->rx_chan))  in usbhsf_dma_init_dt()
1292 			fifo->rx_chan = NULL;  in usbhsf_dma_init_dt()
1296 static void usbhsf_dma_init(struct usbhs_priv *priv, struct usbhs_fifo *fifo,  in usbhsf_dma_init()  argument
1302 		usbhsf_dma_init_dt(dev, fifo, channel);  in usbhsf_dma_init()
1304 		usbhsf_dma_init_pdev(fifo);  in usbhsf_dma_init()
1306 	if (fifo->tx_chan || fifo->rx_chan)  in usbhsf_dma_init()
1308 			 fifo->name,  in usbhsf_dma_init()
1309 			 fifo->tx_chan ? "[TX]" : "    ",  in usbhsf_dma_init()
1310 			 fifo->rx_chan ? "[RX]" : "    ");  in usbhsf_dma_init()
1395 	struct usbhs_fifo *fifo = usbhsf_get_cfifo(priv); /* CFIFO */  in usbhs_fifo_clear_dcp()  local
1397 	/* clear DCP FIFO of transmission */  in usbhs_fifo_clear_dcp()
1398 	if (usbhsf_fifo_select(pipe, fifo, 1) < 0)  in usbhs_fifo_clear_dcp()
1400 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1401 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1403 	/* clear DCP FIFO of reception */  in usbhs_fifo_clear_dcp()
1404 	if (usbhsf_fifo_select(pipe, fifo, 0) < 0)  in usbhs_fifo_clear_dcp()
1406 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1407 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1411  *		fifo init
1440 #define __USBHS_DFIFO_INIT(priv, fifo, channel, fifo_port)		\  argument
1442 	fifo = usbhsf_get_dnfifo(priv, channel);			\
1443 	fifo->name	= "D"#channel"FIFO";				\
1444 	fifo->port	= fifo_port;					\
1445 	fifo->sel	= D##channel##FIFOSEL;				\
1446 	fifo->ctr	= D##channel##FIFOCTR;				\
1447 	fifo->tx_slave.shdma_slave.slave_id =				\
1449 	fifo->rx_slave.shdma_slave.slave_id =				\
1451 	usbhsf_dma_init(priv, fifo, channel);				\
1454 #define USBHS_DFIFO_INIT(priv, fifo, channel)				\  argument
1455 		__USBHS_DFIFO_INIT(priv, fifo, channel, D##channel##FIFO)
1456 #define USBHS_DFIFO_INIT_NO_PORT(priv, fifo, channel)			\  argument
1457 		__USBHS_DFIFO_INIT(priv, fifo, channel, 0)
1461 	struct usbhs_fifo *fifo;  in usbhs_fifo_probe()  local
1464 	fifo = usbhsf_get_cfifo(priv);  in usbhs_fifo_probe()
1465 	fifo->name	= "CFIFO";  in usbhs_fifo_probe()
1466 	fifo->port	= CFIFO;  in usbhs_fifo_probe()
1467 	fifo->sel	= CFIFOSEL;  in usbhs_fifo_probe()
1468 	fifo->ctr	= CFIFOCTR;  in usbhs_fifo_probe()
1471 	USBHS_DFIFO_INIT(priv, fifo, 0);  in usbhs_fifo_probe()
1472 	USBHS_DFIFO_INIT(priv, fifo, 1);  in usbhs_fifo_probe()
1473 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 2);  in usbhs_fifo_probe()
1474 	USBHS_DFIFO_INIT_NO_PORT(priv, fifo, 3);  in usbhs_fifo_probe()
1481 	struct usbhs_fifo *fifo;  in usbhs_fifo_remove()  local
1484 	usbhs_for_each_dfifo(priv, fifo, i)  in usbhs_fifo_remove()
1485 		usbhsf_dma_quit(priv, fifo);  in usbhs_fifo_remove()