22 void usbhs_pkt_init(struct usbhs_pkt *pkt)  in usbhs_pkt_init()  argument
24 	INIT_LIST_HEAD(&pkt->node);  in usbhs_pkt_init()
30 static int usbhsf_null_handle(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_null_handle()  argument
32 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_null_handle()
45 void usbhs_pkt_push(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt,  in usbhs_pkt_push()  argument
47 				 struct usbhs_pkt *pkt),  in usbhs_pkt_push()  argument
67 	list_move_tail(&pkt->node, &pipe->list);  in usbhs_pkt_push()
74 	pkt->pipe	= pipe;  in usbhs_pkt_push()
75 	pkt->buf	= buf;  in usbhs_pkt_push()
76 	pkt->handler	= pipe->handler;  in usbhs_pkt_push()
77 	pkt->length	= len;  in usbhs_pkt_push()
78 	pkt->zero	= zero;  in usbhs_pkt_push()
79 	pkt->actual	= 0;  in usbhs_pkt_push()
80 	pkt->done	= done;  in usbhs_pkt_push()
81 	pkt->sequence	= sequence;  in usbhs_pkt_push()
87 static void __usbhsf_pkt_del(struct usbhs_pkt *pkt)  in __usbhsf_pkt_del()  argument
89 	list_del_init(&pkt->node);  in __usbhsf_pkt_del()
100 					    struct usbhs_pkt *pkt);
103 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map);
104 struct usbhs_pkt *usbhs_pkt_pop(struct usbhs_pipe *pipe, struct usbhs_pkt *pkt)  in usbhs_pkt_pop()  argument
115 	if (!pkt)  in usbhs_pkt_pop()
116 		pkt = __usbhsf_pkt_get(pipe);  in usbhs_pkt_pop()
118 	if (pkt) {  in usbhs_pkt_pop()
122 			chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhs_pkt_pop()
125 			usbhsf_dma_unmap(pkt);  in usbhs_pkt_pop()
130 		__usbhsf_pkt_del(pkt);  in usbhs_pkt_pop()
139 	return pkt;  in usbhs_pkt_pop()
151 	struct usbhs_pkt *pkt;  in usbhsf_pkt_handler()  local
153 	int (*func)(struct usbhs_pkt *pkt, int *is_done);  in usbhsf_pkt_handler()
161 	pkt = __usbhsf_pkt_get(pipe);  in usbhsf_pkt_handler()
162 	if (!pkt)  in usbhsf_pkt_handler()
167 		func = pkt->handler->prepare;  in usbhsf_pkt_handler()
170 		func = pkt->handler->try_run;  in usbhsf_pkt_handler()
173 		func = pkt->handler->dma_done;  in usbhsf_pkt_handler()
181 		ret = func(pkt, &is_done);  in usbhsf_pkt_handler()
184 		__usbhsf_pkt_del(pkt);  in usbhsf_pkt_handler()
191 		pkt->done(priv, pkt);  in usbhsf_pkt_handler()
347 static int usbhs_dcp_dir_switch_to_write(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_write()  argument
349 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_write()
376 static int usbhs_dcp_dir_switch_to_read(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_to_read()  argument
378 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_to_read()
404 static int usbhs_dcp_dir_switch_done(struct usbhs_pkt *pkt, int *is_done)  in usbhs_dcp_dir_switch_done()  argument
406 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dcp_dir_switch_done()
408 	if (pkt->handler == &usbhs_dcp_status_stage_in_handler)  in usbhs_dcp_dir_switch_done()
413 	pkt->actual = pkt->length;  in usbhs_dcp_dir_switch_done()
432 static int usbhsf_dcp_data_stage_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dcp_data_stage_try_push()  argument
434 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_try_push()
441 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dcp_data_stage_try_push()
443 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_try_push()
453 static int usbhsf_dcp_data_stage_prepare_pop(struct usbhs_pkt *pkt,  in usbhsf_dcp_data_stage_prepare_pop()  argument
456 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dcp_data_stage_prepare_pop()
480 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dcp_data_stage_prepare_pop()
482 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dcp_data_stage_prepare_pop()
492 static int usbhsf_pio_try_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_push()  argument
494 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_push()
505 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_pio_try_push()
506 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_pio_try_push()
508 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_pio_try_push()
525 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_push()
526 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_push()
554 	pkt->actual += total_len;  in usbhsf_pio_try_push()
556 	if (pkt->actual < pkt->length)  in usbhsf_pio_try_push()
561 		*is_done = !pkt->zero;	/* send zero packet ? */  in usbhsf_pio_try_push()
575 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_push()
594 static int usbhsf_pio_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_prepare_push()  argument
596 	if (usbhs_pipe_is_running(pkt->pipe))  in usbhsf_pio_prepare_push()
599 	return usbhsf_pio_try_push(pkt, is_done);  in usbhsf_pio_prepare_push()
610 static int usbhsf_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_prepare_pop()  argument
612 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_prepare_pop()
625 	usbhs_pipe_data_sequence(pipe, pkt->sequence);  in usbhsf_prepare_pop()
626 	pkt->sequence = -1; /* -1 sequence will be ignored */  in usbhsf_prepare_pop()
631 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->length);  in usbhsf_prepare_pop()
639 static int usbhsf_pio_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_pio_try_pop()  argument
641 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_pio_try_pop()
663 	buf		= pkt->buf    + pkt->actual;  in usbhsf_pio_try_pop()
664 	len		= pkt->length - pkt->actual;  in usbhsf_pio_try_pop()
673 	pkt->actual += total_len;  in usbhsf_pio_try_pop()
675 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_pio_try_pop()
697 		pkt->zero = 1;  in usbhsf_pio_try_pop()
724 		pkt->length, pkt->actual, *is_done, pkt->zero);  in usbhsf_pio_try_pop()
740 static int usbhsf_ctrl_stage_end(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_ctrl_stage_end()  argument
742 	usbhs_dcp_control_transfer_done(pkt->pipe);  in usbhsf_ctrl_stage_end()
758 					    struct usbhs_pkt *pkt)  in usbhsf_dma_chan_get()  argument
760 	if (&usbhs_fifo_dma_push_handler == pkt->handler)  in usbhsf_dma_chan_get()
763 	if (&usbhs_fifo_dma_pop_handler == pkt->handler)  in usbhsf_dma_chan_get()
770 					      struct usbhs_pkt *pkt)  in usbhsf_get_dma_fifo()  argument
776 		if (usbhsf_dma_chan_get(fifo, pkt) &&  in usbhsf_get_dma_fifo()
795 static int __usbhsf_dma_map_ctrl(struct usbhs_pkt *pkt, int map)  in __usbhsf_dma_map_ctrl()  argument
797 	struct usbhs_pipe *pipe = pkt->pipe;  in __usbhsf_dma_map_ctrl()
801 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in __usbhsf_dma_map_ctrl()
803 	return info->dma_map_ctrl(chan->device->dev, pkt, map);  in __usbhsf_dma_map_ctrl()
807 static void usbhsf_dma_xfer_preparing(struct usbhs_pkt *pkt)  in usbhsf_dma_xfer_preparing()  argument
809 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_xfer_preparing()
821 	chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_xfer_preparing()
824 	desc = dmaengine_prep_slave_single(chan, pkt->dma + pkt->actual,  in usbhsf_dma_xfer_preparing()
825 					pkt->trans, dir,  in usbhsf_dma_xfer_preparing()
833 	pkt->cookie = dmaengine_submit(desc);  in usbhsf_dma_xfer_preparing()
834 	if (pkt->cookie < 0) {  in usbhsf_dma_xfer_preparing()
840 		fifo->name, usbhs_pipe_number(pipe), pkt->length, pkt->zero);  in usbhsf_dma_xfer_preparing()
843 	usbhs_pipe_set_trans_count_if_bulk(pipe, pkt->trans);  in usbhsf_dma_xfer_preparing()
851 	struct usbhs_pkt *pkt = container_of(work, struct usbhs_pkt, work);  in xfer_work()  local
852 	struct usbhs_pipe *pipe = pkt->pipe;  in xfer_work()
857 	usbhsf_dma_xfer_preparing(pkt);  in xfer_work()
864 static int usbhsf_dma_prepare_push(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_push()  argument
866 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_push()
869 	int len = pkt->length - pkt->actual;  in usbhsf_dma_prepare_push()
888 	if ((uintptr_t)(pkt->buf + pkt->actual) & align_mask)  in usbhsf_dma_prepare_push()
896 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_push()
904 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_push()
907 	pkt->trans = len;  in usbhsf_dma_prepare_push()
912 		usbhsf_dma_xfer_preparing(pkt);  in usbhsf_dma_prepare_push()
914 		INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_prepare_push()
915 		schedule_work(&pkt->work);  in usbhsf_dma_prepare_push()
926 	pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_prepare_push()
928 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_push()
931 static int usbhsf_dma_push_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_push_done()  argument
933 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_push_done()
934 	int is_short = pkt->trans % usbhs_pipe_get_maxpacket(pipe);  in usbhsf_dma_push_done()
936 	pkt->actual += pkt->trans;  in usbhsf_dma_push_done()
938 	if (pkt->actual < pkt->length)  in usbhsf_dma_push_done()
943 		*is_done = !pkt->zero;	/* send zero packet? */  in usbhsf_dma_push_done()
948 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_push_done()
953 		pkt->handler = &usbhs_fifo_pio_push_handler;  in usbhsf_dma_push_done()
954 		return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_push_done()
969 static int usbhsf_dma_prepare_pop_with_rx_irq(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_rx_irq()  argument
972 	return usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_prepare_pop_with_rx_irq()
975 static int usbhsf_dma_prepare_pop_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_prepare_pop_with_usb_dmac()  argument
978 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_prepare_pop_with_usb_dmac()
987 	if ((pkt->length < usbhs_get_dparam(priv, pio_dma_border)) ||  in usbhsf_dma_prepare_pop_with_usb_dmac()
991 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
995 	if ((uintptr_t)pkt->buf & (USBHS_USB_DMAC_XFER_SIZE - 1))  in usbhsf_dma_prepare_pop_with_usb_dmac()
1008 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_prepare_pop_with_usb_dmac()
1020 	pkt->trans = pkt->length;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1022 	usbhsf_dma_xfer_preparing(pkt);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1033 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_prepare_pop_with_usb_dmac()
1036 	return pkt->handler->prepare(pkt, is_done);  in usbhsf_dma_prepare_pop_with_usb_dmac()
1039 static int usbhsf_dma_prepare_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_prepare_pop()  argument
1041 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_prepare_pop()
1044 		return usbhsf_dma_prepare_pop_with_usb_dmac(pkt, is_done);  in usbhsf_dma_prepare_pop()
1046 		return usbhsf_dma_prepare_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_prepare_pop()
1049 static int usbhsf_dma_try_pop_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop_with_rx_irq()  argument
1051 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_try_pop_with_rx_irq()
1063 	fifo = usbhsf_get_dma_fifo(priv, pkt);  in usbhsf_dma_try_pop_with_rx_irq()
1067 	if ((uintptr_t)(pkt->buf + pkt->actual) & 0x7) /* 8byte alignment */  in usbhsf_dma_try_pop_with_rx_irq()
1076 	len = min(pkt->length - pkt->actual, len);  in usbhsf_dma_try_pop_with_rx_irq()
1087 	if (usbhsf_dma_map(pkt) < 0)  in usbhsf_dma_try_pop_with_rx_irq()
1099 	pkt->trans = len;  in usbhsf_dma_try_pop_with_rx_irq()
1101 	INIT_WORK(&pkt->work, xfer_work);  in usbhsf_dma_try_pop_with_rx_irq()
1102 	schedule_work(&pkt->work);  in usbhsf_dma_try_pop_with_rx_irq()
1113 	pkt->handler = &usbhs_fifo_pio_pop_handler;  in usbhsf_dma_try_pop_with_rx_irq()
1115 	return pkt->handler->try_run(pkt, is_done);  in usbhsf_dma_try_pop_with_rx_irq()
1118 static int usbhsf_dma_try_pop(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_try_pop()  argument
1120 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_try_pop()
1124 	return usbhsf_dma_try_pop_with_rx_irq(pkt, is_done);  in usbhsf_dma_try_pop()
1127 static int usbhsf_dma_pop_done_with_rx_irq(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done_with_rx_irq()  argument
1129 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_rx_irq()
1133 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_rx_irq()
1136 	pkt->actual += pkt->trans;  in usbhsf_dma_pop_done_with_rx_irq()
1138 	if ((pkt->actual == pkt->length) ||	/* receive all data */  in usbhsf_dma_pop_done_with_rx_irq()
1139 	    (pkt->trans < maxp)) {		/* short packet */  in usbhsf_dma_pop_done_with_rx_irq()
1145 		usbhsf_prepare_pop(pkt, is_done);  in usbhsf_dma_pop_done_with_rx_irq()
1151 static size_t usbhs_dma_calc_received_size(struct usbhs_pkt *pkt,  in usbhs_dma_calc_received_size()  argument
1154 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhs_dma_calc_received_size()
1159 	dmaengine_tx_status(chan, pkt->cookie, &state);  in usbhs_dma_calc_received_size()
1160 	received_size = pkt->length - state.residue;  in usbhs_dma_calc_received_size()
1171 static int usbhsf_dma_pop_done_with_usb_dmac(struct usbhs_pkt *pkt,  in usbhsf_dma_pop_done_with_usb_dmac()  argument
1174 	struct usbhs_pipe *pipe = pkt->pipe;  in usbhsf_dma_pop_done_with_usb_dmac()
1177 	struct dma_chan *chan = usbhsf_dma_chan_get(fifo, pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1189 	pkt->actual = usbhs_dma_calc_received_size(pkt, chan, rcv_len);  in usbhsf_dma_pop_done_with_usb_dmac()
1193 	usbhsf_dma_unmap(pkt);  in usbhsf_dma_pop_done_with_usb_dmac()
1202 static int usbhsf_dma_pop_done(struct usbhs_pkt *pkt, int *is_done)  in usbhsf_dma_pop_done()  argument
1204 	struct usbhs_priv *priv = usbhs_pipe_to_priv(pkt->pipe);  in usbhsf_dma_pop_done()
1207 		return usbhsf_dma_pop_done_with_usb_dmac(pkt, is_done);  in usbhsf_dma_pop_done()
1209 		return usbhsf_dma_pop_done_with_rx_irq(pkt, is_done);  in usbhsf_dma_pop_done()