17 #define usbhsf_fifo_is_busy(f)	((f)->pipe) /* see usbhs_pipe_select_fifo */
32 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
45 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
50 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_pkt_push()
62 	if (!pipe->handler) {  in usbhs_pkt_push()
64 		pipe->handler = &usbhsf_null_handler;  in usbhs_pkt_push()
67 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
74 	pkt->pipe	= pipe;  in usbhs_pkt_push()
76 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
92 struct usbhs_pkt *__usbhsf_pkt_get(struct usbhs_pipe *pipe)  in __usbhsf_pkt_get()  argument
94 	return list_first_entry_or_null(&pipe->list, struct usbhs_pkt, node);  in __usbhsf_pkt_get()
97 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,
104 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
106 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_pkt_pop()
107 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhs_pkt_pop()
113 	usbhs_pipe_disable(pipe);  in usbhs_pkt_pop()
116 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
128 		usbhs_pipe_clear_without_sequence(pipe, 0, 0);  in usbhs_pkt_pop()
134 		usbhsf_fifo_unselect(pipe, fifo);  in usbhs_pkt_pop()
148 static int usbhsf_pkt_handler(struct usbhs_pipe *pipe, int type)  in usbhsf_pkt_handler()  argument
150 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pkt_handler()
161 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
192 		usbhs_pkt_start(pipe);  in usbhsf_pkt_handler()
198 void usbhs_pkt_start(struct usbhs_pipe *pipe)  in usbhs_pkt_start()  argument
200 	usbhsf_pkt_handler(pipe, USBHSF_PKT_PREPARE);  in usbhs_pkt_start()
208 #define usbhsf_irq_callback_ctrl(pipe, status, enable)			\  argument
210 		struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);	\
212 		u16 status = (1 << usbhs_pipe_number(pipe));		\
222 static void usbhsf_tx_irq_ctrl(struct usbhs_pipe *pipe, int enable)  in usbhsf_tx_irq_ctrl()  argument
233 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_tx_irq_ctrl()
234 		usbhsf_irq_empty_ctrl(pipe, enable);  in usbhsf_tx_irq_ctrl()
236 		usbhsf_irq_ready_ctrl(pipe, enable);  in usbhsf_tx_irq_ctrl()
239 static void usbhsf_rx_irq_ctrl(struct usbhs_pipe *pipe, int enable)  in usbhsf_rx_irq_ctrl()  argument
241 	usbhsf_irq_ready_ctrl(pipe, enable);  in usbhsf_rx_irq_ctrl()
247 static void usbhsf_send_terminator(struct usbhs_pipe *pipe,  in usbhsf_send_terminator()  argument
250 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_send_terminator()
265 static void usbhsf_fifo_clear(struct usbhs_pipe *pipe,  in usbhsf_fifo_clear()  argument
268 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_clear()
271 	if (!usbhs_pipe_is_dcp(pipe)) {  in usbhsf_fifo_clear()
277 		if (usbhs_pipe_is_dir_in(pipe))  in usbhsf_fifo_clear()
278 			ret = usbhs_pipe_is_accessible(pipe);  in usbhsf_fifo_clear()
297 static void usbhsf_fifo_unselect(struct usbhs_pipe *pipe,  in usbhsf_fifo_unselect()  argument
300 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_unselect()
302 	usbhs_pipe_select_fifo(pipe, NULL);  in usbhsf_fifo_unselect()
306 static int usbhsf_fifo_select(struct usbhs_pipe *pipe,  in usbhsf_fifo_select()  argument
310 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_fifo_select()
314 	u16 base = usbhs_pipe_number(pipe);	/* CURPIPE */  in usbhsf_fifo_select()
316 	if (usbhs_pipe_is_busy(pipe) ||  in usbhsf_fifo_select()
320 	if (usbhs_pipe_is_dcp(pipe)) {  in usbhsf_fifo_select()
324 			usbhs_dcp_dir_for_host(pipe, write);  in usbhsf_fifo_select()
333 			usbhs_pipe_select_fifo(pipe, fifo);  in usbhsf_fifo_select()
349 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()  local
350 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_dcp_dir_switch_to_write()
355 	usbhs_pipe_disable(pipe);  in usbhs_dcp_dir_switch_to_write()
357 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhs_dcp_dir_switch_to_write()
363 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhs_dcp_dir_switch_to_write()
365 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
366 	usbhsf_send_terminator(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
368 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_write()
370 	usbhsf_tx_irq_ctrl(pipe, 1);  in usbhs_dcp_dir_switch_to_write()
371 	usbhs_pipe_enable(pipe);  in usbhs_dcp_dir_switch_to_write()
378 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()  local
379 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_dcp_dir_switch_to_read()
384 	usbhs_pipe_disable(pipe);  in usbhs_dcp_dir_switch_to_read()
386 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhs_dcp_dir_switch_to_read()
392 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhs_dcp_dir_switch_to_read()
393 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
395 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_dcp_dir_switch_to_read()
397 	usbhsf_rx_irq_ctrl(pipe, 1);  in usbhs_dcp_dir_switch_to_read()
398 	usbhs_pipe_enable(pipe);  in usbhs_dcp_dir_switch_to_read()
406 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()  local
409 		usbhsf_tx_irq_ctrl(pipe, 0);  in usbhs_dcp_dir_switch_done()
411 		usbhsf_rx_irq_ctrl(pipe, 0);  in usbhs_dcp_dir_switch_done()
434 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()  local
436 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhsf_dcp_data_stage_try_push()
456 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()  local
457 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dcp_data_stage_prepare_pop()
460 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dcp_data_stage_prepare_pop()
469 	usbhs_pipe_disable(pipe);  in usbhsf_dcp_data_stage_prepare_pop()
471 	usbhs_pipe_sequence_data1(pipe); /* DATA1 */  in usbhsf_dcp_data_stage_prepare_pop()
473 	usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dcp_data_stage_prepare_pop()
474 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
475 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dcp_data_stage_prepare_pop()
494 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()  local
495 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pio_try_push()
500 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_pio_try_push()
505 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
508 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
510 	ret = usbhsf_fifo_select(pipe, fifo, 1);  in usbhsf_pio_try_push()
514 	ret = usbhs_pipe_is_accessible(pipe);  in usbhsf_pio_try_push()
567 		usbhsf_send_terminator(pipe, fifo);  in usbhsf_pio_try_push()
569 	usbhsf_tx_irq_ctrl(pipe, !*is_done);  in usbhsf_pio_try_push()
570 	usbhs_pipe_running(pipe, !*is_done);  in usbhsf_pio_try_push()
571 	usbhs_pipe_enable(pipe);  in usbhsf_pio_try_push()
574 		usbhs_pipe_number(pipe),  in usbhsf_pio_try_push()
577 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
582 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_push()
588 	usbhsf_tx_irq_ctrl(pipe, 1);  in usbhsf_pio_try_push()
589 	usbhs_pipe_running(pipe, 1);  in usbhsf_pio_try_push()
596 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
612 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()  local
613 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_prepare_pop()
616 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_prepare_pop()
619 	if (usbhs_pipe_is_running(pipe))  in usbhsf_prepare_pop()
625 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
628 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_prepare_pop()
629 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_prepare_pop()
631 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
632 	usbhs_pipe_enable(pipe);  in usbhsf_prepare_pop()
633 	usbhs_pipe_running(pipe, 1);  in usbhsf_prepare_pop()
634 	usbhsf_rx_irq_ctrl(pipe, 1);  in usbhsf_prepare_pop()
641 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()  local
642 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_pio_try_pop()
648 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_pio_try_pop()
653 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_pio_try_pop()
678 		usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_pio_try_pop()
679 		usbhs_pipe_running(pipe, 0);  in usbhsf_pio_try_pop()
686 		if (!usbhs_mod_is_host(priv) && !usbhs_pipe_is_dcp(pipe))  in usbhsf_pio_try_pop()
687 			usbhs_pipe_disable(pipe);	/* disable pipe first */  in usbhsf_pio_try_pop()
698 		usbhsf_fifo_clear(pipe, fifo);  in usbhsf_pio_try_pop()
723 		usbhs_pipe_number(pipe),  in usbhsf_pio_try_pop()
727 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_pio_try_pop()
742 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
786 static void __usbhsf_dma_ctrl(struct usbhs_pipe *pipe,  in __usbhsf_dma_ctrl()  argument
790 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in __usbhsf_dma_ctrl()
797 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()  local
798 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in __usbhsf_dma_map_ctrl()
800 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in __usbhsf_dma_map_ctrl()
809 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_xfer_preparing()  local
811 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_xfer_preparing()
817 	fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_xfer_preparing()
822 	dir = usbhs_pipe_is_dir_in(pipe) ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV;  in usbhsf_dma_xfer_preparing()
831 	desc->callback_param	= pipe;  in usbhsf_dma_xfer_preparing()
840 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in usbhsf_dma_xfer_preparing()
842 	usbhs_pipe_running(pipe, 1);  in usbhsf_dma_xfer_preparing()
843 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in usbhsf_dma_xfer_preparing()
845 	usbhsf_dma_start(pipe, fifo);  in usbhsf_dma_xfer_preparing()
846 	usbhs_pipe_enable(pipe);  in usbhsf_dma_xfer_preparing()
852 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()  local
853 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in xfer_work()
866 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()  local
867 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_prepare_push()
873 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_prepare_push()
878 	    usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))  in usbhsf_dma_prepare_push()
892 	if (usbhs_pipe_is_running(pipe))  in usbhsf_dma_prepare_push()
900 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_push()
909 	usbhsf_tx_irq_ctrl(pipe, 0);  in usbhsf_dma_prepare_push()
921 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_push()
933 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()  local
934 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
945 	usbhs_pipe_running(pipe, !*is_done);  in usbhsf_dma_push_done()
947 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_push_done()
949 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_push_done()
978 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()  local
979 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_prepare_pop_with_usb_dmac()
983 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_prepare_pop_with_usb_dmac()
988 	    usbhs_pipe_type_is(pipe, USB_ENDPOINT_XFER_ISOC))  in usbhsf_dma_prepare_pop_with_usb_dmac()
999 	if (usbhs_pipe_is_running(pipe))  in usbhsf_dma_prepare_pop_with_usb_dmac()
1002 	usbhs_pipe_config_change_bfre(pipe, 1);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1004 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1018 	usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1027 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1034 	usbhs_pipe_config_change_bfre(pipe, 0);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1041 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1051 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()  local
1052 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_try_pop_with_rx_irq()
1056 	if (usbhs_pipe_is_busy(pipe))  in usbhsf_dma_try_pop_with_rx_irq()
1059 	if (usbhs_pipe_is_dcp(pipe))  in usbhsf_dma_try_pop_with_rx_irq()
1070 	ret = usbhsf_fifo_select(pipe, fifo, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1097 	usbhsf_rx_irq_ctrl(pipe, 0);  in usbhsf_dma_try_pop_with_rx_irq()
1107 	usbhsf_fifo_unselect(pipe, fifo);  in usbhsf_dma_try_pop_with_rx_irq()
1120 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1129 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()  local
1130 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_pop_done_with_rx_irq()
1132 	usbhsf_dma_stop(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1134 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_rx_irq()
1141 		usbhs_pipe_running(pipe, 0);  in usbhsf_dma_pop_done_with_rx_irq()
1144 		usbhs_pipe_running(pipe, 0);  in usbhsf_dma_pop_done_with_rx_irq()
1154 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()  local
1157 	int maxp = usbhs_pipe_get_maxpacket(pipe);  in usbhs_dma_calc_received_size()
1174 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()  local
1175 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()
1176 	struct usbhs_fifo *fifo = usbhs_pipe_to_fifo(pipe);  in usbhsf_dma_pop_done_with_usb_dmac()
1185 	usbhs_xxxsts_clear(priv, BRDYSTS, usbhs_pipe_number(pipe));  in usbhsf_dma_pop_done_with_usb_dmac()
1188 	usbhsf_fifo_clear(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1191 	usbhs_pipe_running(pipe, 0);  in usbhsf_dma_pop_done_with_usb_dmac()
1192 	usbhsf_dma_stop(pipe, fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1194 	usbhsf_fifo_unselect(pipe, pipe->fifo);  in usbhsf_dma_pop_done_with_usb_dmac()
1204 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1309 	struct usbhs_pipe *pipe;  in usbhsf_irq_empty()  local
1324 	usbhs_for_each_pipe_with_dcp(pipe, priv, i) {  in usbhsf_irq_empty()
1328 		ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);  in usbhsf_irq_empty()
1339 	struct usbhs_pipe *pipe;  in usbhsf_irq_ready()  local
1354 	usbhs_for_each_pipe_with_dcp(pipe, priv, i) {  in usbhsf_irq_ready()
1358 		ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_TRY_RUN);  in usbhsf_irq_ready()
1368 	struct usbhs_pipe *pipe = arg;  in usbhsf_dma_complete()  local
1369 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhsf_dma_complete()
1373 	ret = usbhsf_pkt_handler(pipe, USBHSF_PKT_DMA_DONE);  in usbhsf_dma_complete()
1376 			usbhs_pipe_number(pipe), ret);  in usbhsf_dma_complete()
1379 void usbhs_fifo_clear_dcp(struct usbhs_pipe *pipe)  in usbhs_fifo_clear_dcp()  argument
1381 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pipe);  in usbhs_fifo_clear_dcp()
1385 	if (usbhsf_fifo_select(pipe, fifo, 1) < 0)  in usbhs_fifo_clear_dcp()
1387 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1388 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1391 	if (usbhsf_fifo_select(pipe, fifo, 0) < 0)  in usbhs_fifo_clear_dcp()
1393 	usbhsf_fifo_clear(pipe, fifo);  in usbhs_fifo_clear_dcp()
1394 	usbhsf_fifo_unselect(pipe, fifo);  in usbhs_fifo_clear_dcp()
1412 	cfifo->pipe	= NULL;  in usbhs_fifo_init()
1414 		dfifo->pipe	= NULL;  in usbhs_fifo_init()