298 void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)  in cdns3_move_deq_to_next_trb()  argument
300 	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;  in cdns3_move_deq_to_next_trb()
301 	int current_trb = priv_req->start_trb;  in cdns3_move_deq_to_next_trb()
303 	while (current_trb != priv_req->end_trb) {  in cdns3_move_deq_to_next_trb()
440 					      struct cdns3_request *priv_req)  in cdns3_wa2_gadget_giveback()  argument
443 	    priv_req->flags & REQUEST_INTERNAL) {  in cdns3_wa2_gadget_giveback()
468 	return &priv_req->request;  in cdns3_wa2_gadget_giveback()
473 			      struct cdns3_request *priv_req)  in cdns3_wa2_gadget_ep_queue()  argument
503 		    !(priv_req->flags & REQUEST_INTERNAL)) {  in cdns3_wa2_gadget_ep_queue()
505 						     &priv_req->request);  in cdns3_wa2_gadget_ep_queue()
509 			list_add_tail(&priv_req->request.list,  in cdns3_wa2_gadget_ep_queue()
511 			cdns3_gadget_giveback(priv_ep, priv_req,  in cdns3_wa2_gadget_ep_queue()
512 					      priv_req->request.status);  in cdns3_wa2_gadget_ep_queue()
531 		if (priv_req->flags & REQUEST_INTERNAL)  in cdns3_wa2_gadget_ep_queue()
532 			list_add_tail(&priv_req->list,  in cdns3_wa2_gadget_ep_queue()
541 	struct cdns3_request *priv_req;  in cdns3_wa2_remove_old_request()  local
546 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_wa2_remove_old_request()
547 		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);  in cdns3_wa2_remove_old_request()
551 		kfree(priv_req->request.buf);  in cdns3_wa2_remove_old_request()
553 					     &priv_req->request);  in cdns3_wa2_remove_old_request()
554 		list_del_init(&priv_req->list);  in cdns3_wa2_remove_old_request()
571 	struct cdns3_request *priv_req;  in cdns3_wa2_descmissing_packet()  local
589 	priv_req = to_cdns3_request(request);  in cdns3_wa2_descmissing_packet()
590 	priv_req->flags |= REQUEST_INTERNAL;  in cdns3_wa2_descmissing_packet()
601 	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,  in cdns3_wa2_descmissing_packet()
605 	if (!priv_req->request.buf) {  in cdns3_wa2_descmissing_packet()
610 	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;  in cdns3_wa2_descmissing_packet()
611 	priv_ep->descmis_req = priv_req;  in cdns3_wa2_descmissing_packet()
635 			   struct cdns3_request *priv_req,  in cdns3_gadget_giveback()  argument
639 	struct usb_request *request = &priv_req->request;  in cdns3_gadget_giveback()
649 	if ((priv_req->flags & REQUEST_UNALIGNED) &&  in cdns3_gadget_giveback()
651 		memcpy(request->buf, priv_req->aligned_buf->buf,  in cdns3_gadget_giveback()
654 	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);  in cdns3_gadget_giveback()
655 	trace_cdns3_gadget_giveback(priv_req);  in cdns3_gadget_giveback()
659 						    priv_req);  in cdns3_gadget_giveback()
722 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)  in cdns3_prepare_aligned_request_buf()  argument
724 	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;  in cdns3_prepare_aligned_request_buf()
729 	if (!((uintptr_t)priv_req->request.buf & 0x7))  in cdns3_prepare_aligned_request_buf()
732 	buf = priv_req->aligned_buf;  in cdns3_prepare_aligned_request_buf()
734 	if (!buf || priv_req->request.length > buf->size) {  in cdns3_prepare_aligned_request_buf()
739 		buf->size = priv_req->request.length;  in cdns3_prepare_aligned_request_buf()
750 		if (priv_req->aligned_buf) {  in cdns3_prepare_aligned_request_buf()
751 			trace_cdns3_free_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
752 			priv_req->aligned_buf->in_use = 0;  in cdns3_prepare_aligned_request_buf()
758 		priv_req->aligned_buf = buf;  in cdns3_prepare_aligned_request_buf()
765 		memcpy(buf->buf, priv_req->request.buf,  in cdns3_prepare_aligned_request_buf()
766 		       priv_req->request.length);  in cdns3_prepare_aligned_request_buf()
769 	priv_req->flags |= REQUEST_UNALIGNED;  in cdns3_prepare_aligned_request_buf()
770 	trace_cdns3_prepare_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
820 	struct cdns3_request *priv_req;  in cdns3_ep_run_transfer()  local
840 	priv_req = to_cdns3_request(request);  in cdns3_ep_run_transfer()
846 	if (priv_req->flags & REQUEST_UNALIGNED)  in cdns3_ep_run_transfer()
847 		trb_dma = priv_req->aligned_buf->dma;  in cdns3_ep_run_transfer()
852 	priv_req->start_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
853 	priv_req->trb = trb;  in cdns3_ep_run_transfer()
941 			priv_req->trb->control = control;  in cdns3_ep_run_transfer()
945 		priv_req->end_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
950 	trb = priv_req->trb;  in cdns3_ep_run_transfer()
952 	priv_req->flags |= REQUEST_PENDING;  in cdns3_ep_run_transfer()
969 	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);  in cdns3_ep_run_transfer()
995 					priv_req->start_trb * TRB_SIZE),  in cdns3_ep_run_transfer()
1089 				  struct cdns3_request *priv_req)  in cdns3_request_handled()  argument
1092 	struct cdns3_trb *trb = priv_req->trb;  in cdns3_request_handled()
1100 	trb = &priv_ep->trb_pool[priv_req->start_trb];  in cdns3_request_handled()
1122 		if (priv_req->end_trb >= priv_ep->dequeue &&  in cdns3_request_handled()
1123 		    priv_req->end_trb < current_index)  in cdns3_request_handled()
1126 		if (priv_req->end_trb  < current_index ||  in cdns3_request_handled()
1127 		    priv_req->end_trb >= priv_ep->dequeue)  in cdns3_request_handled()
1132 	trace_cdns3_request_handled(priv_req, current_index, handled);  in cdns3_request_handled()
1140 	struct cdns3_request *priv_req;  in cdns3_transfer_completed()  local
1146 		priv_req = to_cdns3_request(request);  in cdns3_transfer_completed()
1153 			cdns3_move_deq_to_next_trb(priv_req);  in cdns3_transfer_completed()
1161 		if (!cdns3_request_handled(priv_ep, priv_req))  in cdns3_transfer_completed()
1167 		if (trb != priv_req->trb)  in cdns3_transfer_completed()
1170 				 priv_req->trb, trb);  in cdns3_transfer_completed()
1173 		cdns3_move_deq_to_next_trb(priv_req);  in cdns3_transfer_completed()
1174 		cdns3_gadget_giveback(priv_ep, priv_req, 0);  in cdns3_transfer_completed()
1739 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_alloc_request()  local
1741 	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);  in cdns3_gadget_ep_alloc_request()
1742 	if (!priv_req)  in cdns3_gadget_ep_alloc_request()
1745 	priv_req->priv_ep = priv_ep;  in cdns3_gadget_ep_alloc_request()
1747 	trace_cdns3_alloc_request(priv_req);  in cdns3_gadget_ep_alloc_request()
1748 	return &priv_req->request;  in cdns3_gadget_ep_alloc_request()
1759 	struct cdns3_request *priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_free_request()  local
1761 	if (priv_req->aligned_buf)  in cdns3_gadget_ep_free_request()
1762 		priv_req->aligned_buf->in_use = 0;  in cdns3_gadget_ep_free_request()
1764 	trace_cdns3_free_request(priv_req);  in cdns3_gadget_ep_free_request()
1765 	kfree(priv_req);  in cdns3_gadget_ep_free_request()
1891 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_disable()  local
1945 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_gadget_ep_disable()
1947 		kfree(priv_req->request.buf);  in cdns3_gadget_ep_disable()
1949 					     &priv_req->request);  in cdns3_gadget_ep_disable()
1950 		list_del_init(&priv_req->list);  in cdns3_gadget_ep_disable()
1985 	struct cdns3_request *priv_req;  in __cdns3_gadget_ep_queue()  local
1990 	priv_req = to_cdns3_request(request);  in __cdns3_gadget_ep_queue()
1991 	trace_cdns3_ep_queue(priv_req);  in __cdns3_gadget_ep_queue()
1995 						priv_req);  in __cdns3_gadget_ep_queue()
2001 	ret = cdns3_prepare_aligned_request_buf(priv_req);  in __cdns3_gadget_ep_queue()
2045 		struct cdns3_request *priv_req;  in cdns3_gadget_ep_queue()  local
2051 		priv_req = to_cdns3_request(zlp_request);  in cdns3_gadget_ep_queue()
2052 		priv_req->flags |= REQUEST_ZLP;  in cdns3_gadget_ep_queue()
2076 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_dequeue()  local
2087 	priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_dequeue()
2089 	trace_cdns3_ep_dequeue(priv_req);  in cdns3_gadget_ep_dequeue()
2110 	link_trb = priv_req->trb;  in cdns3_gadget_ep_dequeue()
2115 					      (priv_req->start_trb * TRB_SIZE));  in cdns3_gadget_ep_dequeue()
2119 		if (priv_ep->wa1_trb == priv_req->trb)  in cdns3_gadget_ep_dequeue()
2123 	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);  in cdns3_gadget_ep_dequeue()