355 static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)  in cdns3_move_deq_to_next_trb()  argument
357 	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;  in cdns3_move_deq_to_next_trb()
358 	int current_trb = priv_req->start_trb;  in cdns3_move_deq_to_next_trb()
360 	while (current_trb != priv_req->end_trb) {  in cdns3_move_deq_to_next_trb()
424 		struct cdns3_request *priv_req;  in cdns3_start_all_request()  local
427 		priv_req = to_cdns3_request(request);  in cdns3_start_all_request()
428 		if ((priv_req->flags & REQUEST_INTERNAL) ||  in cdns3_start_all_request()
539 						     struct cdns3_request *priv_req)  in cdns3_wa2_gadget_giveback()  argument
542 	    priv_req->flags & REQUEST_INTERNAL) {  in cdns3_wa2_gadget_giveback()
574 	return &priv_req->request;  in cdns3_wa2_gadget_giveback()
579 				     struct cdns3_request *priv_req)  in cdns3_wa2_gadget_ep_queue()  argument
609 		    !(priv_req->flags & REQUEST_INTERNAL)) {  in cdns3_wa2_gadget_ep_queue()
611 						     &priv_req->request);  in cdns3_wa2_gadget_ep_queue()
615 			list_add_tail(&priv_req->request.list,  in cdns3_wa2_gadget_ep_queue()
617 			cdns3_gadget_giveback(priv_ep, priv_req,  in cdns3_wa2_gadget_ep_queue()
618 					      priv_req->request.status);  in cdns3_wa2_gadget_ep_queue()
637 		if (priv_req->flags & REQUEST_INTERNAL)  in cdns3_wa2_gadget_ep_queue()
638 			list_add_tail(&priv_req->list,  in cdns3_wa2_gadget_ep_queue()
647 	struct cdns3_request *priv_req;  in cdns3_wa2_remove_old_request()  local
652 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_wa2_remove_old_request()
653 		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);  in cdns3_wa2_remove_old_request()
657 		kfree(priv_req->request.buf);  in cdns3_wa2_remove_old_request()
659 					     &priv_req->request);  in cdns3_wa2_remove_old_request()
660 		list_del_init(&priv_req->list);  in cdns3_wa2_remove_old_request()
677 	struct cdns3_request *priv_req;  in cdns3_wa2_descmissing_packet()  local
704 	priv_req = to_cdns3_request(request);  in cdns3_wa2_descmissing_packet()
705 	priv_req->flags |= REQUEST_INTERNAL;  in cdns3_wa2_descmissing_packet()
716 	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,  in cdns3_wa2_descmissing_packet()
720 	if (!priv_req->request.buf) {  in cdns3_wa2_descmissing_packet()
725 	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;  in cdns3_wa2_descmissing_packet()
726 	priv_ep->descmis_req = priv_req;  in cdns3_wa2_descmissing_packet()
822 			   struct cdns3_request *priv_req,  in cdns3_gadget_giveback()  argument
826 	struct usb_request *request = &priv_req->request;  in cdns3_gadget_giveback()
836 	if ((priv_req->flags & REQUEST_UNALIGNED) &&  in cdns3_gadget_giveback()
838 		memcpy(request->buf, priv_req->aligned_buf->buf,  in cdns3_gadget_giveback()
841 	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);  in cdns3_gadget_giveback()
843 	priv_req->finished_trb = 0;  in cdns3_gadget_giveback()
844 	trace_cdns3_gadget_giveback(priv_req);  in cdns3_gadget_giveback()
848 						    priv_req);  in cdns3_gadget_giveback()
911 static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)  in cdns3_prepare_aligned_request_buf()  argument
913 	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;  in cdns3_prepare_aligned_request_buf()
918 	if (!((uintptr_t)priv_req->request.buf & 0x7))  in cdns3_prepare_aligned_request_buf()
921 	buf = priv_req->aligned_buf;  in cdns3_prepare_aligned_request_buf()
923 	if (!buf || priv_req->request.length > buf->size) {  in cdns3_prepare_aligned_request_buf()
928 		buf->size = priv_req->request.length;  in cdns3_prepare_aligned_request_buf()
939 		if (priv_req->aligned_buf) {  in cdns3_prepare_aligned_request_buf()
940 			trace_cdns3_free_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
941 			priv_req->aligned_buf->in_use = 0;  in cdns3_prepare_aligned_request_buf()
947 		priv_req->aligned_buf = buf;  in cdns3_prepare_aligned_request_buf()
954 		memcpy(buf->buf, priv_req->request.buf,  in cdns3_prepare_aligned_request_buf()
955 		       priv_req->request.length);  in cdns3_prepare_aligned_request_buf()
958 	priv_req->flags |= REQUEST_UNALIGNED;  in cdns3_prepare_aligned_request_buf()
959 	trace_cdns3_prepare_aligned_request(priv_req);  in cdns3_prepare_aligned_request_buf()
1003 	struct cdns3_request *priv_req;  in cdns3_ep_run_stream_transfer()  local
1012 	priv_req = to_cdns3_request(request);  in cdns3_ep_run_stream_transfer()
1018 	if (priv_req->flags & REQUEST_UNALIGNED)  in cdns3_ep_run_stream_transfer()
1019 		trb_dma = priv_req->aligned_buf->dma;  in cdns3_ep_run_stream_transfer()
1025 	priv_req->start_trb = priv_ep->enqueue;  in cdns3_ep_run_stream_transfer()
1026 	priv_req->end_trb = priv_req->start_trb;  in cdns3_ep_run_stream_transfer()
1027 	priv_req->trb = trb;  in cdns3_ep_run_stream_transfer()
1032 		  TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;  in cdns3_ep_run_stream_transfer()
1055 	priv_req->flags |= REQUEST_PENDING;  in cdns3_ep_run_stream_transfer()
1059 	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);  in cdns3_ep_run_stream_transfer()
1089 		priv_ep->last_stream_id = priv_req->request.stream_id;  in cdns3_ep_run_stream_transfer()
1091 		writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |  in cdns3_ep_run_stream_transfer()
1115 	struct cdns3_request *priv_req;  in cdns3_ep_run_transfer()  local
1139 	priv_req = to_cdns3_request(request);  in cdns3_ep_run_transfer()
1145 	if (priv_req->flags & REQUEST_UNALIGNED)  in cdns3_ep_run_transfer()
1146 		trb_dma = priv_req->aligned_buf->dma;  in cdns3_ep_run_transfer()
1151 	priv_req->start_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
1152 	priv_req->trb = trb;  in cdns3_ep_run_transfer()
1247 			priv_req->trb->control = cpu_to_le32(control);  in cdns3_ep_run_transfer()
1260 		priv_req->end_trb = priv_ep->enqueue;  in cdns3_ep_run_transfer()
1266 	trb = priv_req->trb;  in cdns3_ep_run_transfer()
1268 	priv_req->flags |= REQUEST_PENDING;  in cdns3_ep_run_transfer()
1269 	priv_req->num_of_trb = num_trb;  in cdns3_ep_run_transfer()
1317 		trace_cdns3_prepare_trb(priv_ep, priv_req->trb);  in cdns3_ep_run_transfer()
1344 					priv_req->start_trb * TRB_SIZE),  in cdns3_ep_run_transfer()
1430 				  struct cdns3_request *priv_req)  in cdns3_trb_handled()  argument
1442 	if (priv_req->start_trb < priv_req->end_trb) {  in cdns3_trb_handled()
1443 		if (priv_ep->dequeue > priv_req->end_trb)  in cdns3_trb_handled()
1446 		if (priv_ep->dequeue < priv_req->start_trb)  in cdns3_trb_handled()
1450 	if ((priv_req->start_trb > priv_req->end_trb) &&  in cdns3_trb_handled()
1451 		(priv_ep->dequeue > priv_req->end_trb) &&  in cdns3_trb_handled()
1452 		(priv_ep->dequeue < priv_req->start_trb))  in cdns3_trb_handled()
1455 	if ((priv_req->start_trb == priv_req->end_trb) &&  in cdns3_trb_handled()
1456 		(priv_ep->dequeue != priv_req->end_trb))  in cdns3_trb_handled()
1487 	trace_cdns3_request_handled(priv_req, current_index, handled);  in cdns3_trb_handled()
1495 	struct cdns3_request *priv_req;  in cdns3_transfer_completed()  local
1503 		priv_req = to_cdns3_request(request);  in cdns3_transfer_completed()
1510 			cdns3_move_deq_to_next_trb(priv_req);  in cdns3_transfer_completed()
1519 			while (cdns3_trb_handled(priv_ep, priv_req)) {  in cdns3_transfer_completed()
1520 				priv_req->finished_trb++;  in cdns3_transfer_completed()
1521 				if (priv_req->finished_trb >= priv_req->num_of_trb)  in cdns3_transfer_completed()
1531 				if (priv_req->num_of_trb > 1 &&  in cdns3_transfer_completed()
1539 				cdns3_gadget_giveback(priv_ep, priv_req, 0);  in cdns3_transfer_completed()
1558 			if (trb != priv_req->trb)  in cdns3_transfer_completed()
1561 					 priv_req->trb, trb);  in cdns3_transfer_completed()
1568 				cdns3_gadget_giveback(priv_ep, priv_req, 0);  in cdns3_transfer_completed()
2235 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_alloc_request()  local
2237 	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);  in cdns3_gadget_ep_alloc_request()
2238 	if (!priv_req)  in cdns3_gadget_ep_alloc_request()
2241 	priv_req->priv_ep = priv_ep;  in cdns3_gadget_ep_alloc_request()
2243 	trace_cdns3_alloc_request(priv_req);  in cdns3_gadget_ep_alloc_request()
2244 	return &priv_req->request;  in cdns3_gadget_ep_alloc_request()
2255 	struct cdns3_request *priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_free_request()  local
2257 	if (priv_req->aligned_buf)  in cdns3_gadget_ep_free_request()
2258 		priv_req->aligned_buf->in_use = 0;  in cdns3_gadget_ep_free_request()
2260 	trace_cdns3_free_request(priv_req);  in cdns3_gadget_ep_free_request()
2261 	kfree(priv_req);  in cdns3_gadget_ep_free_request()
2408 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_disable()  local
2462 		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);  in cdns3_gadget_ep_disable()
2464 		kfree(priv_req->request.buf);  in cdns3_gadget_ep_disable()
2466 					     &priv_req->request);  in cdns3_gadget_ep_disable()
2467 		list_del_init(&priv_req->list);  in cdns3_gadget_ep_disable()
2503 	struct cdns3_request *priv_req;  in __cdns3_gadget_ep_queue()  local
2508 	priv_req = to_cdns3_request(request);  in __cdns3_gadget_ep_queue()
2509 	trace_cdns3_ep_queue(priv_req);  in __cdns3_gadget_ep_queue()
2513 						priv_req);  in __cdns3_gadget_ep_queue()
2519 	ret = cdns3_prepare_aligned_request_buf(priv_req);  in __cdns3_gadget_ep_queue()
2571 		struct cdns3_request *priv_req;  in cdns3_gadget_ep_queue()  local
2577 		priv_req = to_cdns3_request(zlp_request);  in cdns3_gadget_ep_queue()
2578 		priv_req->flags |= REQUEST_ZLP;  in cdns3_gadget_ep_queue()
2602 	struct cdns3_request *priv_req;  in cdns3_gadget_ep_dequeue()  local
2613 	priv_req = to_cdns3_request(request);  in cdns3_gadget_ep_dequeue()
2615 	trace_cdns3_ep_dequeue(priv_req);  in cdns3_gadget_ep_dequeue()
2636 	link_trb = priv_req->trb;  in cdns3_gadget_ep_dequeue()
2641 			((priv_req->end_trb + 1) * TRB_SIZE)));  in cdns3_gadget_ep_dequeue()
2645 		if (priv_ep->wa1_trb == priv_req->trb)  in cdns3_gadget_ep_dequeue()
2649 	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);  in cdns3_gadget_ep_dequeue()
2686 	struct cdns3_request *priv_req;  in __cdns3_gadget_ep_clear_halt()  local
2695 		priv_req = to_cdns3_request(request);  in __cdns3_gadget_ep_clear_halt()
2696 		trb = priv_req->trb;  in __cdns3_gadget_ep_clear_halt()