22 void usbhs_pkt_init(struct usbhs_pkt *pkt)  in usbhs_pkt_init()  argument
24 	INIT_LIST_HEAD(&pkt->node);  in usbhs_pkt_init()
30 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_null_handle()  argument
32 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
45 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
47 				 struct usbhs_pkt *pkt),  in usbhs_pkt_push()  argument
67 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
74 	pkt->pipe	= pipe;  in usbhs_pkt_push()
75 	pkt->buf	= buf;  in usbhs_pkt_push()
76 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
77 	pkt->length	= len;  in usbhs_pkt_push()
78 	pkt->zero	= zero;  in usbhs_pkt_push()
79 	pkt->actual	= 0;  in usbhs_pkt_push()
80 	pkt->done	= done;  in usbhs_pkt_push()
81 	pkt->sequence	= sequence;  in usbhs_pkt_push()
87 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)  in __usbhsf_pkt_del()  argument
89 	list_del_init(&pkt->node);  in __usbhsf_pkt_del()
100 					    struct usbhs_pkt *pkt);
103 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map);
104 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
115 	if (!pkt)  in usbhs_pkt_pop()
116 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
118 	if (pkt) {  in usbhs_pkt_pop()
122 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
125 			usbhsf_dma_unmap(pkt);  in usbhs_pkt_pop()
130 		__usbhsf_pkt_del(pkt);  in usbhs_pkt_pop()
139 	return pkt;  in usbhs_pkt_pop()
151 	struct usbhs_pkt *pkt;  in usbhsf_pkt_handler()  local
153 	int (*func)(struct usbhs_pkt *pkt, int *is_done);  in usbhsf_pkt_handler()
161 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
162 	if (!pkt)  in usbhsf_pkt_handler()
167 		func = pkt->handler->prepare;  in usbhsf_pkt_handler()
170 		func = pkt->handler->try_run;  in usbhsf_pkt_handler()
173 		func = pkt->handler->dma_done;  in usbhsf_pkt_handler()
181 		ret = func(pkt, &is_done);  in usbhsf_pkt_handler()
184 		__usbhsf_pkt_del(pkt);  in usbhsf_pkt_handler()
191 		pkt->done(priv, pkt);  in usbhsf_pkt_handler()
350 static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_write()  argument
352 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()
379 static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_read()  argument
381 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()
407 static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_done()  argument
409 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()
411 	if (pkt->handler == &usbhs_dcp_status_stage_in_handler)  in usbhs_dcp_dir_switch_done()
416 	pkt->actual = pkt->length;  in usbhs_dcp_dir_switch_done()
435 static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dcp_data_stage_try_push()  argument
437 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()
444 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dcp_data_stage_try_push()
446 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_try_push()
456 static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt,  in usbhsf_dcp_data_stage_prepare_pop()  argument
459 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()
483 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dcp_data_stage_prepare_pop()
485 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_prepare_pop()
495 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_push()  argument
497 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()
508 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
509 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_pio_try_push()
511 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
528 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_push()
529 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_push()
552 	pkt->actual += total_len;  in usbhsf_pio_try_push()
554 	if (pkt->actual < pkt->length)  in usbhsf_pio_try_push()
559 		*is_done = !pkt->zero;	/* send zero packet ? */  in usbhsf_pio_try_push()
573 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_push()
592 static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_prepare_push()  argument
594 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
597 	return usbhsf_pio_try_push(pkt, is_done);  in usbhsf_pio_prepare_push()
608 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_prepare_pop()  argument
610 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()
623 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
624 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_prepare_pop()
629 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
637 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_pop()  argument
639 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()
661 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_pop()
662 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_pop()
671 	pkt->actual += total_len;  in usbhsf_pio_try_pop()
673 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_pio_try_pop()
695 		pkt->zero = 1;  in usbhsf_pio_try_pop()
722 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_pop()
738 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_ctrl_stage_end()  argument
740 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
756 					    struct usbhs_pkt *pkt)  in usbhsf_dma_chan_get()  argument
758 	if (&usbhs_fifo_dma_push_handler == pkt->handler)  in usbhsf_dma_chan_get()
761 	if (&usbhs_fifo_dma_pop_handler == pkt->handler)  in usbhsf_dma_chan_get()
768 					      struct usbhs_pkt *pkt)  in usbhsf_get_dma_fifo()  argument
774 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
793 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)  in __usbhsf_dma_map_ctrl()  argument
795 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()
799 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in __usbhsf_dma_map_ctrl()
801 	return info->dma_map_ctrl(chan->device->dev, pkt, map);  in __usbhsf_dma_map_ctrl()
807 	struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work);  in xfer_work()  local
808 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()
822 	chan = usbhsf_dma_chan_get(fifo, pkt);  in xfer_work()
825 	desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual,  in xfer_work()
826 					pkt->trans, dir,  in xfer_work()
834 	pkt->cookie = dmaengine_submit(desc);  in xfer_work()
835 	if (pkt->cookie < 0) {  in xfer_work()
841 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in xfer_work()
844 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in xfer_work()
856 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_push()  argument
858 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()
861 	int len = pkt->length - pkt->actual;  in usbhsf_dma_prepare_push()
880 	if ((uintptr_t)(pkt->buf + pkt->actual) & align_mask)  in usbhsf_dma_prepare_push()
888 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
896 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_push()
899 	pkt->trans = len;  in usbhsf_dma_prepare_push()
902 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_push()
903 	schedule_work(&pkt->work);  in usbhsf_dma_prepare_push()
913 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_prepare_push()
915 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_push()
918 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_push_done()  argument
920 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()
921 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
923 	pkt->actual += pkt->trans;  in usbhsf_dma_push_done()
925 	if (pkt->actual < pkt->length)  in usbhsf_dma_push_done()
930 		*is_done = !pkt->zero;	/* send zero packet? */  in usbhsf_dma_push_done()
935 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_push_done()
940 		pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_push_done()
941 		return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_push_done()
956 static int usbhsf_dma_prepare_pop_with_rx_irq(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_rx_irq()  argument
959 	return usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_prepare_pop_with_rx_irq()
962 static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_usb_dmac()  argument
965 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()
974 	if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||  in usbhsf_dma_prepare_pop_with_usb_dmac()
978 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
982 	if ((uintptr_t)pkt->buf & (USBHS_USB_DMAC_XFER_SIZE - 1))  in usbhsf_dma_prepare_pop_with_usb_dmac()
995 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_pop_with_usb_dmac()
1007 	pkt->trans = pkt->length;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1009 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1010 	schedule_work(&pkt->work);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1021 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1024 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1027 static int usbhsf_dma_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_pop()  argument
1029 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1032 		return usbhsf_dma_prepare_pop_with_usb_dmac(pkt, is_done);  in usbhsf_dma_prepare_pop()
1034 		return usbhsf_dma_prepare_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_prepare_pop()
1037 static int usbhsf_dma_try_pop_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop_with_rx_irq()  argument
1039 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()
1051 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1055 	if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */  in usbhsf_dma_try_pop_with_rx_irq()
1064 	len = min(pkt->length - pkt->actual, len);  in usbhsf_dma_try_pop_with_rx_irq()
1075 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_try_pop_with_rx_irq()
1087 	pkt->trans = len;  in usbhsf_dma_try_pop_with_rx_irq()
1089 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_try_pop_with_rx_irq()
1090 	schedule_work(&pkt->work);  in usbhsf_dma_try_pop_with_rx_irq()
1101 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_try_pop_with_rx_irq()
1103 	return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_try_pop_with_rx_irq()
1106 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop()  argument
1108 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1112 	return usbhsf_dma_try_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_try_pop()
1115 static int usbhsf_dma_pop_done_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done_with_rx_irq()  argument
1117 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()
1121 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_rx_irq()
1124 	pkt->actual += pkt->trans;  in usbhsf_dma_pop_done_with_rx_irq()
1126 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_dma_pop_done_with_rx_irq()
1127 	    (pkt->trans < maxp)) {		/* short packet */  in usbhsf_dma_pop_done_with_rx_irq()
1133 		usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_pop_done_with_rx_irq()
1139 static size_t usbhs_dma_calc_received_size(struct usbhs_pkt *pkt,  in usbhs_dma_calc_received_size()  argument
1142 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()
1147 	dmaengine_tx_status(chan, pkt->cookie, &state);  in usbhs_dma_calc_received_size()
1148 	received_size = pkt->length - state.residue;  in usbhs_dma_calc_received_size()
1159 static int usbhsf_dma_pop_done_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_pop_done_with_usb_dmac()  argument
1162 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()
1165 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1177 	pkt->actual = usbhs_dma_calc_received_size(pkt, chan, rcv_len);  in usbhsf_dma_pop_done_with_usb_dmac()
1181 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1190 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done()  argument
1192 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1195 		return usbhsf_dma_pop_done_with_usb_dmac(pkt, is_done);  in usbhsf_dma_pop_done()
1197 		return usbhsf_dma_pop_done_with_rx_irq(pkt, is_done);  in usbhsf_dma_pop_done()