340 				       struct cdnsp_request *preq)  in cdnsp_request_to_transfer_ring()  argument
342 	return cdnsp_get_transfer_ring(pdev, preq->pep,  in cdnsp_request_to_transfer_ring()
343 				       preq->request.stream_id);  in cdnsp_request_to_transfer_ring()
602 	struct cdnsp_request *preq;  in cdnsp_unmap_td_bounce_buffer()  local
608 	preq = td->preq;  in cdnsp_unmap_td_bounce_buffer()
610 	trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs,  in cdnsp_unmap_td_bounce_buffer()
613 	if (!preq->direction) {  in cdnsp_unmap_td_bounce_buffer()
623 	len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs,  in cdnsp_unmap_td_bounce_buffer()
691 			 struct cdnsp_request *preq,  in cdnsp_remove_request()  argument
705 	trace_cdnsp_remove_request_td(preq);  in cdnsp_remove_request()
707 	cur_td = &preq->td;  in cdnsp_remove_request()
708 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_remove_request()
715 	hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id);  in cdnsp_remove_request()
722 		cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id,  in cdnsp_remove_request()
745 	cdnsp_gadget_giveback(pep, cur_td->preq, status);  in cdnsp_remove_request()
894 	struct cdnsp_request *preq = td->preq;  in cdnsp_td_cleanup()  local
903 	if (preq->request.actual > preq->request.length) {  in cdnsp_td_cleanup()
904 		preq->request.actual = 0;  in cdnsp_td_cleanup()
910 	preq->pep->stream_info.td_count--;  in cdnsp_td_cleanup()
912 	cdnsp_gadget_giveback(preq->pep, preq, *status);  in cdnsp_td_cleanup()
1014 		td->preq->request.actual = td->preq->request.length - remaining;  in cdnsp_process_ctrl_td()
1019 		td->preq->request.actual = td->preq->request.length;  in cdnsp_process_ctrl_td()
1045 	struct cdnsp_request *preq = td->preq;  in cdnsp_process_isoc_td()  local
1057 	requested = preq->request.length;  in cdnsp_process_isoc_td()
1062 		preq->request.status = 0;  in cdnsp_process_isoc_td()
1065 		preq->request.status = 0;  in cdnsp_process_isoc_td()
1070 		preq->request.status = -EOVERFLOW;  in cdnsp_process_isoc_td()
1077 		preq->request.status  = 0;  in cdnsp_process_isoc_td()
1086 		preq->request.status = -1;  in cdnsp_process_isoc_td()
1097 	td->preq->request.actual += td_length;  in cdnsp_process_isoc_td()
1111 	td->preq->request.status = -EXDEV;  in cdnsp_skip_isoc_td()
1112 	td->preq->request.actual = 0;  in cdnsp_skip_isoc_td()
1141 	requested = td->preq->request.length;  in cdnsp_process_bulk_intr_td()
1149 		td->preq->request.actual = remaining;  in cdnsp_process_bulk_intr_td()
1163 	td->preq->request.actual = ep_trb_len;  in cdnsp_process_bulk_intr_td()
1348 		desc = td->preq->pep->endpoint.desc;  in cdnsp_handle_tx_event()
1653 				  struct cdnsp_request *preq,  in cdnsp_prepare_transfer()  argument
1659 	ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,  in cdnsp_prepare_transfer()
1660 					  preq->request.stream_id);  in cdnsp_prepare_transfer()
1665 				 GET_EP_CTX_STATE(preq->pep->out_ctx),  in cdnsp_prepare_transfer()
1670 	INIT_LIST_HEAD(&preq->td.td_list);  in cdnsp_prepare_transfer()
1671 	preq->td.preq = preq;  in cdnsp_prepare_transfer()
1674 	list_add_tail(&preq->td.td_list, &ep_ring->td_list);  in cdnsp_prepare_transfer()
1676 	preq->pep->stream_info.td_count++;  in cdnsp_prepare_transfer()
1678 	preq->td.start_seg = ep_ring->enq_seg;  in cdnsp_prepare_transfer()
1679 	preq->td.first_trb = ep_ring->enqueue;  in cdnsp_prepare_transfer()
1696 static unsigned int count_trbs_needed(struct cdnsp_request *preq)  in count_trbs_needed()  argument
1698 	return cdnsp_count_trbs(preq->request.dma, preq->request.length);  in count_trbs_needed()
1701 static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq)  in count_sg_trbs_needed()  argument
1706 	full_len = preq->request.length;  in count_sg_trbs_needed()
1708 	for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) {  in count_sg_trbs_needed()
1720 static unsigned int count_isoc_trbs_needed(struct cdnsp_request *preq)  in count_isoc_trbs_needed()  argument
1722 	return cdnsp_count_trbs(preq->request.dma, preq->request.length);  in count_isoc_trbs_needed()
1725 static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total)  in cdnsp_check_trb_math()  argument
1727 	if (running_total != preq->request.length)  in cdnsp_check_trb_math()
1728 		dev_err(preq->pep->pdev->dev,  in cdnsp_check_trb_math()
1731 			preq->pep->name, running_total,  in cdnsp_check_trb_math()
1732 			preq->request.length, preq->request.actual);  in cdnsp_check_trb_math()
1756 			      struct cdnsp_request *preq,  in cdnsp_td_remainder()  argument
1766 	maxp = usb_endpoint_maxp(preq->pep->endpoint.desc);  in cdnsp_td_remainder()
1774 			  struct cdnsp_request *preq, u32 enqd_len,  in cdnsp_align_td()  argument
1782 	max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);  in cdnsp_align_td()
1792 		trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len,  in cdnsp_align_td()
1804 	if (new_buff_len > (preq->request.length - enqd_len))  in cdnsp_align_td()
1805 		new_buff_len = (preq->request.length - enqd_len);  in cdnsp_align_td()
1808 	if (preq->direction) {  in cdnsp_align_td()
1809 		sg_pcopy_to_buffer(preq->request.sg,  in cdnsp_align_td()
1810 				   preq->request.num_mapped_sgs,  in cdnsp_align_td()
1830 	trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma,  in cdnsp_align_td()
1840 int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)  in cdnsp_queue_bulk_tx()  argument
1857 	ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_queue_bulk_tx()
1861 	full_len = preq->request.length;  in cdnsp_queue_bulk_tx()
1863 	if (preq->request.num_sgs) {  in cdnsp_queue_bulk_tx()
1864 		num_sgs = preq->request.num_sgs;  in cdnsp_queue_bulk_tx()
1865 		sg = preq->request.sg;  in cdnsp_queue_bulk_tx()
1868 		num_trbs = count_sg_trbs_needed(preq);  in cdnsp_queue_bulk_tx()
1870 		num_trbs = count_trbs_needed(preq);  in cdnsp_queue_bulk_tx()
1871 		addr = (u64)preq->request.dma;  in cdnsp_queue_bulk_tx()
1875 	pep = preq->pep;  in cdnsp_queue_bulk_tx()
1878 	if (preq->request.zero && preq->request.length &&  in cdnsp_queue_bulk_tx()
1884 	ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);  in cdnsp_queue_bulk_tx()
1924 				if (cdnsp_align_td(pdev, preq, enqd_len,  in cdnsp_queue_bulk_tx()
1929 					preq->td.bounce_seg = ring->enq_seg;  in cdnsp_queue_bulk_tx()
1941 			preq->td.last_trb = ring->enqueue;  in cdnsp_queue_bulk_tx()
1945 		if (!preq->direction)  in cdnsp_queue_bulk_tx()
1950 					       full_len, preq,  in cdnsp_queue_bulk_tx()
1979 	cdnsp_check_trb_math(preq, enqd_len);  in cdnsp_queue_bulk_tx()
1980 	ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id,  in cdnsp_queue_bulk_tx()
1984 		preq->td.drbl = 1;  in cdnsp_queue_bulk_tx()
1989 int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)  in cdnsp_queue_ctrl_tx()  argument
1992 	struct cdnsp_ep *pep = preq->pep;  in cdnsp_queue_ctrl_tx()
1997 	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);  in cdnsp_queue_ctrl_tx()
2004 	ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);  in cdnsp_queue_ctrl_tx()
2014 	if (preq->request.length > 0) {  in cdnsp_queue_ctrl_tx()
2015 		remainder = cdnsp_td_remainder(pdev, 0, preq->request.length,  in cdnsp_queue_ctrl_tx()
2016 					       preq->request.length, preq, 1);  in cdnsp_queue_ctrl_tx()
2018 		length_field = TRB_LEN(preq->request.length) |  in cdnsp_queue_ctrl_tx()
2025 				lower_32_bits(preq->request.dma),  in cdnsp_queue_ctrl_tx()
2026 				upper_32_bits(preq->request.dma), length_field,  in cdnsp_queue_ctrl_tx()
2035 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_ctrl_tx()
2038 	if (preq->request.length == 0)  in cdnsp_queue_ctrl_tx()
2043 	if (preq->request.length > 0 && pdev->ep0_expect_in)  in cdnsp_queue_ctrl_tx()
2057 	cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id);  in cdnsp_queue_ctrl_tx()
2105 					  struct cdnsp_request *preq,  in cdnsp_get_burst_count()  argument
2113 	max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;  in cdnsp_get_burst_count()
2127 					  struct cdnsp_request *preq,  in cdnsp_get_last_burst_packet_count()  argument
2135 		max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;  in cdnsp_get_last_burst_packet_count()
2155 			       struct cdnsp_request *preq)  in cdnsp_queue_isoc_tx()  argument
2170 	ep_ring = preq->pep->ring;  in cdnsp_queue_isoc_tx()
2173 	td_len = preq->request.length;  in cdnsp_queue_isoc_tx()
2174 	addr = (u64)preq->request.dma;  in cdnsp_queue_isoc_tx()
2177 	max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);  in cdnsp_queue_isoc_tx()
2184 	burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count);  in cdnsp_queue_isoc_tx()
2185 	last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq,  in cdnsp_queue_isoc_tx()
2187 	trbs_per_td = count_isoc_trbs_needed(preq);  in cdnsp_queue_isoc_tx()
2189 	ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td);  in cdnsp_queue_isoc_tx()
2215 					       trb_buff_len, td_len, preq,  in cdnsp_queue_isoc_tx()
2229 		if (usb_endpoint_dir_out(preq->pep->endpoint.desc))  in cdnsp_queue_isoc_tx()
2238 			preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2258 	cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id,  in cdnsp_queue_isoc_tx()
2265 	list_del_init(&preq->td.td_list);  in cdnsp_queue_isoc_tx()
2275 	preq->td.last_trb = ep_ring->enqueue;  in cdnsp_queue_isoc_tx()
2277 	cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);  in cdnsp_queue_isoc_tx()
2280 	ep_ring->enqueue = preq->td.first_trb;  in cdnsp_queue_isoc_tx()
2281 	ep_ring->enq_seg = preq->td.start_seg;  in cdnsp_queue_isoc_tx()
2287 				struct cdnsp_request *preq)  in cdnsp_queue_isoc_tx_prepare()  argument
2294 	ep_ring = preq->pep->ring;  in cdnsp_queue_isoc_tx_prepare()
2295 	ep_state = GET_EP_CTX_STATE(preq->pep->out_ctx);  in cdnsp_queue_isoc_tx_prepare()
2296 	num_trbs = count_isoc_trbs_needed(preq);  in cdnsp_queue_isoc_tx_prepare()
2307 	return cdnsp_queue_isoc_tx(pdev, preq);  in cdnsp_queue_isoc_tx_prepare()