Lines Matching refs:hs_ep
116 static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_incr_frame_num() argument
118 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_incr_frame_num()
124 hs_ep->target_frame += hs_ep->interval; in dwc2_gadget_incr_frame_num()
125 if (hs_ep->target_frame > limit) { in dwc2_gadget_incr_frame_num()
126 hs_ep->frame_overrun = true; in dwc2_gadget_incr_frame_num()
127 hs_ep->target_frame &= limit; in dwc2_gadget_incr_frame_num()
129 hs_ep->frame_overrun = false; in dwc2_gadget_incr_frame_num()
143 static inline void dwc2_gadget_dec_frame_num_by_one(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_dec_frame_num_by_one() argument
145 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_dec_frame_num_by_one()
151 if (hs_ep->target_frame) in dwc2_gadget_dec_frame_num_by_one()
152 hs_ep->target_frame -= 1; in dwc2_gadget_dec_frame_num_by_one()
154 hs_ep->target_frame = limit; in dwc2_gadget_dec_frame_num_by_one()
417 static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep) in is_ep_periodic() argument
419 return hs_ep->periodic; in is_ep_periodic()
432 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_unmap_dma() argument
437 usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir); in dwc2_hsotg_unmap_dma()
505 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_write_fifo() argument
508 bool periodic = is_ep_periodic(hs_ep); in dwc2_hsotg_write_fifo()
511 int to_write = hs_ep->size_loaded; in dwc2_hsotg_write_fifo()
517 to_write -= (buf_pos - hs_ep->last_load); in dwc2_hsotg_write_fifo()
524 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index)); in dwc2_hsotg_write_fifo()
539 if (hs_ep->fifo_load != 0) { in dwc2_hsotg_write_fifo()
546 hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size); in dwc2_hsotg_write_fifo()
549 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_write_fifo()
552 can_write = hs_ep->fifo_load - size_done; in dwc2_hsotg_write_fifo()
556 can_write = hs_ep->fifo_size - can_write; in dwc2_hsotg_write_fifo()
564 } else if (hsotg->dedicated_fifos && hs_ep->index != 0) { in dwc2_hsotg_write_fifo()
566 DTXFSTS(hs_ep->fifo_index)); in dwc2_hsotg_write_fifo()
584 max_transfer = hs_ep->ep.maxpacket * hs_ep->mc; in dwc2_hsotg_write_fifo()
648 hs_ep->total_data += to_write; in dwc2_hsotg_write_fifo()
651 hs_ep->fifo_load += to_write; in dwc2_hsotg_write_fifo()
656 dwc2_writel_rep(hsotg, EPFIFO(hs_ep->index), data, to_write); in dwc2_hsotg_write_fifo()
668 static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep) in get_ep_limit() argument
670 int index = hs_ep->index; in get_ep_limit()
679 if (hs_ep->dir_in) in get_ep_limit()
694 if ((maxpkt * hs_ep->ep.maxpacket) < maxsize) in get_ep_limit()
695 maxsize = maxpkt * hs_ep->ep.maxpacket; in get_ep_limit()
726 static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_get_chain_limit() argument
728 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_chain_limit()
729 int is_isoc = hs_ep->isochronous; in dwc2_gadget_get_chain_limit()
731 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_chain_limit()
732 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_chain_limit()
735 maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT : in dwc2_gadget_get_chain_limit()
742 if (hs_ep->index) in dwc2_gadget_get_chain_limit()
766 static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask) in dwc2_gadget_get_desc_params() argument
768 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_desc_params()
769 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_desc_params()
770 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_desc_params()
773 if (!hs_ep->index && !dir_in) { in dwc2_gadget_get_desc_params()
776 } else if (hs_ep->isochronous) { in dwc2_gadget_get_desc_params()
793 if (hs_ep->index) in dwc2_gadget_get_desc_params()
802 static void dwc2_gadget_fill_nonisoc_xfer_ddma_one(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_fill_nonisoc_xfer_ddma_one() argument
808 int dir_in = hs_ep->dir_in; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
809 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
815 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
817 hs_ep->desc_count = (len / maxsize) + in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
820 hs_ep->desc_count = 1; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
822 for (i = 0; i < hs_ep->desc_count; ++i) { in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
828 if (!hs_ep->index && !dir_in) in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
843 ((hs_ep->send_zlp && true_last) ? in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
868 static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_config_nonisoc_xfer_ddma() argument
873 struct dwc2_dma_desc *desc = hs_ep->desc_list; in dwc2_gadget_config_nonisoc_xfer_ddma()
878 if (hs_ep->req) in dwc2_gadget_config_nonisoc_xfer_ddma()
879 ureq = &hs_ep->req->req; in dwc2_gadget_config_nonisoc_xfer_ddma()
883 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc, in dwc2_gadget_config_nonisoc_xfer_ddma()
890 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc, in dwc2_gadget_config_nonisoc_xfer_ddma()
893 desc_count += hs_ep->desc_count; in dwc2_gadget_config_nonisoc_xfer_ddma()
896 hs_ep->desc_count = desc_count; in dwc2_gadget_config_nonisoc_xfer_ddma()
910 static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_fill_isoc_desc() argument
914 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_fill_isoc_desc()
919 dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_gadget_fill_isoc_desc()
921 index = hs_ep->next_desc; in dwc2_gadget_fill_isoc_desc()
922 desc = &hs_ep->desc_list[index]; in dwc2_gadget_fill_isoc_desc()
932 if (hs_ep->next_desc) in dwc2_gadget_fill_isoc_desc()
933 hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L; in dwc2_gadget_fill_isoc_desc()
936 __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index); in dwc2_gadget_fill_isoc_desc()
945 if (hs_ep->dir_in) { in dwc2_gadget_fill_isoc_desc()
947 pid = DIV_ROUND_UP(len, hs_ep->ep.maxpacket); in dwc2_gadget_fill_isoc_desc()
952 ((len % hs_ep->ep.maxpacket) ? in dwc2_gadget_fill_isoc_desc()
954 ((hs_ep->target_frame << in dwc2_gadget_fill_isoc_desc()
963 if (hs_ep->dir_in) in dwc2_gadget_fill_isoc_desc()
964 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_fill_isoc_desc()
967 hs_ep->next_desc++; in dwc2_gadget_fill_isoc_desc()
968 if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC) in dwc2_gadget_fill_isoc_desc()
969 hs_ep->next_desc = 0; in dwc2_gadget_fill_isoc_desc()
981 static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_start_isoc_ddma() argument
983 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_start_isoc_ddma()
985 int index = hs_ep->index; in dwc2_gadget_start_isoc_ddma()
993 if (list_empty(&hs_ep->queue)) { in dwc2_gadget_start_isoc_ddma()
994 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_gadget_start_isoc_ddma()
1001 desc = &hs_ep->desc_list[i]; in dwc2_gadget_start_isoc_ddma()
1007 hs_ep->next_desc = 0; in dwc2_gadget_start_isoc_ddma()
1008 list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) { in dwc2_gadget_start_isoc_ddma()
1015 ret = dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr, in dwc2_gadget_start_isoc_ddma()
1021 hs_ep->compl_desc = 0; in dwc2_gadget_start_isoc_ddma()
1022 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); in dwc2_gadget_start_isoc_ddma()
1023 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); in dwc2_gadget_start_isoc_ddma()
1026 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg); in dwc2_gadget_start_isoc_ddma()
1033 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep);
1035 struct dwc2_hsotg_ep *hs_ep,
1050 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_start_req() argument
1055 int index = hs_ep->index; in dwc2_hsotg_start_req()
1056 int dir_in = hs_ep->dir_in; in dwc2_hsotg_start_req()
1067 if (hs_ep->req && !continuing) { in dwc2_hsotg_start_req()
1071 } else if (hs_ep->req != hs_req && continuing) { in dwc2_hsotg_start_req()
1085 hs_ep->dir_in ? "in" : "out"); in dwc2_hsotg_start_req()
1100 maxreq = get_ep_limit(hs_ep); in dwc2_hsotg_start_req()
1102 maxreq = dwc2_gadget_get_chain_limit(hs_ep); in dwc2_hsotg_start_req()
1105 int round = maxreq % hs_ep->ep.maxpacket; in dwc2_hsotg_start_req()
1118 packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket); in dwc2_hsotg_start_req()
1123 if (hs_ep->isochronous) in dwc2_hsotg_start_req()
1136 if ((ureq->length >= hs_ep->ep.maxpacket) && in dwc2_hsotg_start_req()
1137 !(ureq->length % hs_ep->ep.maxpacket)) in dwc2_hsotg_start_req()
1138 hs_ep->send_zlp = 1; in dwc2_hsotg_start_req()
1148 hs_ep->req = hs_req; in dwc2_hsotg_start_req()
1152 u32 mps = hs_ep->ep.maxpacket; in dwc2_hsotg_start_req()
1166 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset, in dwc2_hsotg_start_req()
1170 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg); in dwc2_hsotg_start_req()
1173 __func__, (u32)hs_ep->desc_list_dma, dma_reg); in dwc2_hsotg_start_req()
1191 if (hs_ep->isochronous) { in dwc2_hsotg_start_req()
1192 if (!dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_hsotg_start_req()
1193 if (hs_ep->interval == 1) { in dwc2_hsotg_start_req()
1194 if (hs_ep->target_frame & 0x1) in dwc2_hsotg_start_req()
1201 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_hsotg_start_req()
1203 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA); in dwc2_hsotg_start_req()
1224 hs_ep->size_loaded = length; in dwc2_hsotg_start_req()
1225 hs_ep->last_load = ureq->actual; in dwc2_hsotg_start_req()
1229 hs_ep->fifo_load = 0; in dwc2_hsotg_start_req()
1231 dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); in dwc2_hsotg_start_req()
1249 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1); in dwc2_hsotg_start_req()
1265 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_map_dma() argument
1270 hs_ep->map_dir = hs_ep->dir_in; in dwc2_hsotg_map_dma()
1271 ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); in dwc2_hsotg_map_dma()
1285 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_handle_unaligned_buf_start() argument
1297 hs_ep->ep.name, req_buf, hs_req->req.length); in dwc2_hsotg_handle_unaligned_buf_start()
1311 if (hs_ep->dir_in) in dwc2_hsotg_handle_unaligned_buf_start()
1318 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_handle_unaligned_buf_complete() argument
1326 hs_ep->ep.name, hs_req->req.status, hs_req->req.actual); in dwc2_hsotg_handle_unaligned_buf_complete()
1329 if (!hs_ep->dir_in && !hs_req->req.status) in dwc2_hsotg_handle_unaligned_buf_complete()
1347 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_target_frame_elapsed() argument
1349 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_target_frame_elapsed()
1350 u32 target_frame = hs_ep->target_frame; in dwc2_gadget_target_frame_elapsed()
1352 bool frame_overrun = hs_ep->frame_overrun; in dwc2_gadget_target_frame_elapsed()
1377 struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_set_ep0_desc_chain() argument
1382 hs_ep->desc_list = hsotg->setup_desc[0]; in dwc2_gadget_set_ep0_desc_chain()
1383 hs_ep->desc_list_dma = hsotg->setup_desc_dma[0]; in dwc2_gadget_set_ep0_desc_chain()
1387 hs_ep->desc_list = hsotg->ctrl_in_desc; in dwc2_gadget_set_ep0_desc_chain()
1388 hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma; in dwc2_gadget_set_ep0_desc_chain()
1391 hs_ep->desc_list = hsotg->ctrl_out_desc; in dwc2_gadget_set_ep0_desc_chain()
1392 hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma; in dwc2_gadget_set_ep0_desc_chain()
1407 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_queue() local
1408 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_queue()
1432 if (hs_ep->isochronous && in dwc2_hsotg_ep_queue()
1433 req->length > (hs_ep->mc * hs_ep->ep.maxpacket)) { in dwc2_hsotg_ep_queue()
1441 if (using_desc_dma(hs) && hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1442 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_hsotg_ep_queue()
1443 if (hs_ep->dir_in && req->length > maxsize) { in dwc2_hsotg_ep_queue()
1449 if (!hs_ep->dir_in && req->length > hs_ep->ep.maxpacket) { in dwc2_hsotg_ep_queue()
1451 req->length, hs_ep->ep.maxpacket); in dwc2_hsotg_ep_queue()
1456 ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req); in dwc2_hsotg_ep_queue()
1462 ret = dwc2_hsotg_map_dma(hs, hs_ep, req); in dwc2_hsotg_ep_queue()
1467 if (using_desc_dma(hs) && !hs_ep->index) { in dwc2_hsotg_ep_queue()
1468 ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep); in dwc2_hsotg_ep_queue()
1473 first = list_empty(&hs_ep->queue); in dwc2_hsotg_ep_queue()
1474 list_add_tail(&hs_req->queue, &hs_ep->queue); in dwc2_hsotg_ep_queue()
1482 if (using_desc_dma(hs) && hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1483 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) { in dwc2_hsotg_ep_queue()
1490 dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr, in dwc2_hsotg_ep_queue()
1497 if (!hs_ep->index && !req->length && !hs_ep->dir_in && in dwc2_hsotg_ep_queue()
1499 hs_ep->dir_in = 1; in dwc2_hsotg_ep_queue()
1502 if (!hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1503 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); in dwc2_hsotg_ep_queue()
1509 while (dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_hsotg_ep_queue()
1510 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_ep_queue()
1517 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) in dwc2_hsotg_ep_queue()
1518 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); in dwc2_hsotg_ep_queue()
1526 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_queue_lock() local
1527 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_queue_lock()
1557 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_complete_oursetup() local
1558 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_complete_oursetup()
1729 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep) in get_ep_head() argument
1731 return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req, in get_ep_head()
1743 static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_start_next_request() argument
1745 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_start_next_request()
1746 int dir_in = hs_ep->dir_in; in dwc2_gadget_start_next_request()
1749 if (!list_empty(&hs_ep->queue)) { in dwc2_gadget_start_next_request()
1750 hs_req = get_ep_head(hs_ep); in dwc2_gadget_start_next_request()
1751 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false); in dwc2_gadget_start_next_request()
1754 if (!hs_ep->isochronous) in dwc2_gadget_start_next_request()
2012 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_complete_setup() local
2013 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_complete_setup()
2068 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_program_zlp() argument
2071 u8 index = hs_ep->index; in dwc2_hsotg_program_zlp()
2072 u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); in dwc2_hsotg_program_zlp()
2073 u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); in dwc2_hsotg_program_zlp()
2075 if (hs_ep->dir_in) in dwc2_hsotg_program_zlp()
2083 dma_addr_t dma = hs_ep->desc_list_dma; in dwc2_hsotg_program_zlp()
2086 dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep); in dwc2_hsotg_program_zlp()
2088 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0); in dwc2_hsotg_program_zlp()
2116 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_complete_request() argument
2126 hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete); in dwc2_hsotg_complete_request()
2137 dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req); in dwc2_hsotg_complete_request()
2139 dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req); in dwc2_hsotg_complete_request()
2141 hs_ep->req = NULL; in dwc2_hsotg_complete_request()
2151 usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req); in dwc2_hsotg_complete_request()
2156 if (using_desc_dma(hsotg) && hs_ep->isochronous) in dwc2_hsotg_complete_request()
2165 if (!hs_ep->req && result >= 0) in dwc2_hsotg_complete_request()
2166 dwc2_gadget_start_next_request(hs_ep); in dwc2_hsotg_complete_request()
2178 static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_complete_isoc_request_ddma() argument
2180 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_complete_isoc_request_ddma()
2186 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status; in dwc2_gadget_complete_isoc_request_ddma()
2192 hs_req = get_ep_head(hs_ep); in dwc2_gadget_complete_isoc_request_ddma()
2202 mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK : in dwc2_gadget_complete_isoc_request_ddma()
2210 if (!hs_ep->dir_in && ureq->length & 0x3) in dwc2_gadget_complete_isoc_request_ddma()
2219 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_gadget_complete_isoc_request_ddma()
2221 hs_ep->compl_desc++; in dwc2_gadget_complete_isoc_request_ddma()
2222 if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1)) in dwc2_gadget_complete_isoc_request_ddma()
2223 hs_ep->compl_desc = 0; in dwc2_gadget_complete_isoc_request_ddma()
2224 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status; in dwc2_gadget_complete_isoc_request_ddma()
2237 static void dwc2_gadget_handle_isoc_bna(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_isoc_bna() argument
2239 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_isoc_bna()
2241 if (!hs_ep->dir_in) in dwc2_gadget_handle_isoc_bna()
2243 dwc2_hsotg_complete_request(hsotg, hs_ep, get_ep_head(hs_ep), 0); in dwc2_gadget_handle_isoc_bna()
2245 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_gadget_handle_isoc_bna()
2246 hs_ep->next_desc = 0; in dwc2_gadget_handle_isoc_bna()
2247 hs_ep->compl_desc = 0; in dwc2_gadget_handle_isoc_bna()
2262 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx]; in dwc2_hsotg_rx_data() local
2263 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_rx_data()
2300 hs_ep->total_data += to_read; in dwc2_hsotg_rx_data()
2340 static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_get_xfersize_ddma() argument
2342 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_xfersize_ddma()
2343 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_get_xfersize_ddma()
2346 struct dwc2_dma_desc *desc = hs_ep->desc_list; in dwc2_gadget_get_xfersize_ddma()
2349 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_xfersize_ddma()
2350 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_xfersize_ddma()
2356 if (hs_ep->index) in dwc2_gadget_get_xfersize_ddma()
2360 for (i = 0; i < hs_ep->desc_count; ++i) { in dwc2_gadget_get_xfersize_ddma()
2390 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum]; in dwc2_hsotg_handle_outdone() local
2391 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_handle_outdone()
2403 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_handle_outdone()
2409 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); in dwc2_hsotg_handle_outdone()
2423 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_handle_outdone()
2424 size_done += hs_ep->last_load; in dwc2_hsotg_handle_outdone()
2431 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); in dwc2_hsotg_handle_outdone()
2454 if (!using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_handle_outdone()
2455 req->frame_number = hs_ep->target_frame; in dwc2_hsotg_handle_outdone()
2456 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_handle_outdone()
2459 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result); in dwc2_hsotg_handle_outdone()
2582 struct dwc2_hsotg_ep *hs_ep; in dwc2_hsotg_set_ep_maxpacket() local
2585 hs_ep = index_to_ep(hsotg, ep, dir_in); in dwc2_hsotg_set_ep_maxpacket()
2586 if (!hs_ep) in dwc2_hsotg_set_ep_maxpacket()
2596 hs_ep->ep.maxpacket = mps_bytes; in dwc2_hsotg_set_ep_maxpacket()
2597 hs_ep->mc = 1; in dwc2_hsotg_set_ep_maxpacket()
2601 hs_ep->mc = mc; in dwc2_hsotg_set_ep_maxpacket()
2604 hs_ep->ep.maxpacket = mps; in dwc2_hsotg_set_ep_maxpacket()
2650 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_trytx() argument
2652 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_trytx()
2654 if (!hs_ep->dir_in || !hs_req) { in dwc2_hsotg_trytx()
2659 if (hs_ep->index != 0) in dwc2_hsotg_trytx()
2660 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, in dwc2_hsotg_trytx()
2661 hs_ep->dir_in, 0); in dwc2_hsotg_trytx()
2667 hs_ep->index); in dwc2_hsotg_trytx()
2668 return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); in dwc2_hsotg_trytx()
2683 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_complete_in() argument
2685 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_complete_in()
2686 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index)); in dwc2_hsotg_complete_in()
2695 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) { in dwc2_hsotg_complete_in()
2702 hs_ep->dir_in = 0; in dwc2_hsotg_complete_in()
2704 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_complete_in()
2730 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); in dwc2_hsotg_complete_in()
2738 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_complete_in()
2739 size_done += hs_ep->last_load; in dwc2_hsotg_complete_in()
2751 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); in dwc2_hsotg_complete_in()
2756 if (hs_ep->send_zlp) { in dwc2_hsotg_complete_in()
2757 hs_ep->send_zlp = 0; in dwc2_hsotg_complete_in()
2759 dwc2_hsotg_program_zlp(hsotg, hs_ep); in dwc2_hsotg_complete_in()
2765 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) { in dwc2_hsotg_complete_in()
2772 if (!using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_complete_in()
2773 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_hsotg_complete_in()
2774 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_complete_in()
2777 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_complete_in()
2821 static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_ep_disabled() argument
2823 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_ep_disabled()
2825 unsigned char idx = hs_ep->index; in dwc2_gadget_handle_ep_disabled()
2826 int dir_in = hs_ep->dir_in; in dwc2_gadget_handle_ep_disabled()
2835 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index); in dwc2_gadget_handle_ep_disabled()
2851 if (!hs_ep->isochronous) in dwc2_gadget_handle_ep_disabled()
2854 if (list_empty(&hs_ep->queue)) { in dwc2_gadget_handle_ep_disabled()
2856 __func__, hs_ep); in dwc2_gadget_handle_ep_disabled()
2861 hs_req = get_ep_head(hs_ep); in dwc2_gadget_handle_ep_disabled()
2863 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_gadget_handle_ep_disabled()
2865 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, in dwc2_gadget_handle_ep_disabled()
2868 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_ep_disabled()
2871 } while (dwc2_gadget_target_frame_elapsed(hs_ep)); in dwc2_gadget_handle_ep_disabled()
2937 struct dwc2_hsotg_ep *hs_ep);
2953 static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_nak() argument
2955 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_nak()
2957 int dir_in = hs_ep->dir_in; in dwc2_gadget_handle_nak()
2960 if (!dir_in || !hs_ep->isochronous) in dwc2_gadget_handle_nak()
2963 if (hs_ep->target_frame == TARGET_FRAME_INITIAL) { in dwc2_gadget_handle_nak()
2966 hs_ep->target_frame = hsotg->frame_number; in dwc2_gadget_handle_nak()
2967 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
2976 hs_ep->target_frame &= ~hs_ep->interval + 1; in dwc2_gadget_handle_nak()
2981 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
2982 dwc2_gadget_dec_frame_num_by_one(hs_ep); in dwc2_gadget_handle_nak()
2985 dwc2_gadget_start_isoc_ddma(hs_ep); in dwc2_gadget_handle_nak()
2989 hs_ep->target_frame = hsotg->frame_number; in dwc2_gadget_handle_nak()
2990 if (hs_ep->interval > 1) { in dwc2_gadget_handle_nak()
2992 DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
2993 if (hs_ep->target_frame & 0x1) in dwc2_gadget_handle_nak()
2998 dwc2_writel(hsotg, ctrl, DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
3005 ctrl = dwc2_readl(hsotg, DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
3007 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); in dwc2_gadget_handle_nak()
3009 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index); in dwc2_gadget_handle_nak()
3011 while (dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_nak()
3012 hs_req = get_ep_head(hs_ep); in dwc2_gadget_handle_nak()
3014 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_gadget_handle_nak()
3016 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA); in dwc2_gadget_handle_nak()
3019 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
3024 if (!hs_ep->req) in dwc2_gadget_handle_nak()
3025 dwc2_gadget_start_next_request(hs_ep); in dwc2_gadget_handle_nak()
3039 struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in); in dwc2_hsotg_epint() local
3050 if (!hs_ep) { in dwc2_hsotg_epint()
3069 if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in && in dwc2_hsotg_epint()
3080 if (using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_epint()
3081 dwc2_gadget_complete_isoc_request_ddma(hs_ep); in dwc2_hsotg_epint()
3088 if (!hs_ep->isochronous || !(ints & DXEPINT_NAKINTRPT)) in dwc2_hsotg_epint()
3089 dwc2_hsotg_complete_in(hsotg, hs_ep); in dwc2_hsotg_epint()
3091 if (idx == 0 && !hs_ep->req) in dwc2_hsotg_epint()
3098 if (!hs_ep->isochronous || !(ints & DXEPINT_OUTTKNEPDIS)) in dwc2_hsotg_epint()
3104 dwc2_gadget_handle_ep_disabled(hs_ep); in dwc2_hsotg_epint()
3107 dwc2_gadget_handle_out_token_ep_disabled(hs_ep); in dwc2_hsotg_epint()
3110 dwc2_gadget_handle_nak(hs_ep); in dwc2_hsotg_epint()
3162 if (hs_ep->isochronous) in dwc2_hsotg_epint()
3163 dwc2_gadget_handle_isoc_bna(hs_ep); in dwc2_hsotg_epint()
3166 if (dir_in && !hs_ep->isochronous) { in dwc2_hsotg_epint()
3185 dwc2_hsotg_trytx(hsotg, hs_ep); in dwc2_hsotg_epint()
3616 struct dwc2_hsotg_ep *hs_ep; in dwc2_gadget_handle_incomplete_isoc_in() local
3626 hs_ep = hsotg->eps_in[idx]; in dwc2_gadget_handle_incomplete_isoc_in()
3628 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous) in dwc2_gadget_handle_incomplete_isoc_in()
3633 dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_incomplete_isoc_in()
3663 struct dwc2_hsotg_ep *hs_ep; in dwc2_gadget_handle_incomplete_isoc_out() local
3672 hs_ep = hsotg->eps_out[idx]; in dwc2_gadget_handle_incomplete_isoc_out()
3674 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous) in dwc2_gadget_handle_incomplete_isoc_out()
3679 dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_incomplete_isoc_out()
3835 struct dwc2_hsotg_ep *hs_ep; in dwc2_hsotg_irq() local
3846 hs_ep = hsotg->eps_out[idx]; in dwc2_hsotg_irq()
3854 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) { in dwc2_hsotg_irq()
3862 if (hs_ep->halted) { in dwc2_hsotg_irq()
3906 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_ep_stop_xfr() argument
3911 epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : in dwc2_hsotg_ep_stop_xfr()
3912 DOEPCTL(hs_ep->index); in dwc2_hsotg_ep_stop_xfr()
3913 epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : in dwc2_hsotg_ep_stop_xfr()
3914 DOEPINT(hs_ep->index); in dwc2_hsotg_ep_stop_xfr()
3917 hs_ep->name); in dwc2_hsotg_ep_stop_xfr()
3919 if (hs_ep->dir_in) { in dwc2_hsotg_ep_stop_xfr()
3920 if (hsotg->dedicated_fifos || hs_ep->periodic) { in dwc2_hsotg_ep_stop_xfr()
3977 if (hs_ep->dir_in) { in dwc2_hsotg_ep_stop_xfr()
3980 if (hsotg->dedicated_fifos || hs_ep->periodic) in dwc2_hsotg_ep_stop_xfr()
3981 fifo_index = hs_ep->fifo_index; in dwc2_hsotg_ep_stop_xfr()
3989 if (!hsotg->dedicated_fifos && !hs_ep->periodic) in dwc2_hsotg_ep_stop_xfr()
4008 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_enable() local
4009 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_enable()
4011 unsigned int index = hs_ep->index; in dwc2_hsotg_ep_enable()
4035 if (dir_in != hs_ep->dir_in) { in dwc2_hsotg_ep_enable()
4074 if (using_desc_dma(hsotg) && !hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4075 hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev, in dwc2_hsotg_ep_enable()
4077 &hs_ep->desc_list_dma, GFP_ATOMIC); in dwc2_hsotg_ep_enable()
4078 if (!hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4096 dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in); in dwc2_hsotg_ep_enable()
4099 hs_ep->isochronous = 0; in dwc2_hsotg_ep_enable()
4100 hs_ep->periodic = 0; in dwc2_hsotg_ep_enable()
4101 hs_ep->halted = 0; in dwc2_hsotg_ep_enable()
4102 hs_ep->wedged = 0; in dwc2_hsotg_ep_enable()
4103 hs_ep->interval = desc->bInterval; in dwc2_hsotg_ep_enable()
4109 hs_ep->isochronous = 1; in dwc2_hsotg_ep_enable()
4110 hs_ep->interval = 1 << (desc->bInterval - 1); in dwc2_hsotg_ep_enable()
4111 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_hsotg_ep_enable()
4112 hs_ep->next_desc = 0; in dwc2_hsotg_ep_enable()
4113 hs_ep->compl_desc = 0; in dwc2_hsotg_ep_enable()
4115 hs_ep->periodic = 1; in dwc2_hsotg_ep_enable()
4133 hs_ep->periodic = 1; in dwc2_hsotg_ep_enable()
4136 hs_ep->interval = 1 << (desc->bInterval - 1); in dwc2_hsotg_ep_enable()
4155 size = hs_ep->ep.maxpacket * hs_ep->mc; in dwc2_hsotg_ep_enable()
4178 hs_ep->fifo_index = fifo_index; in dwc2_hsotg_ep_enable()
4179 hs_ep->fifo_size = fifo_size; in dwc2_hsotg_ep_enable()
4183 if (index && !hs_ep->isochronous) in dwc2_hsotg_ep_enable()
4193 hs_ep->isochronous && dir_in) { in dwc2_hsotg_ep_enable()
4221 if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4224 hs_ep->desc_list, hs_ep->desc_list_dma); in dwc2_hsotg_ep_enable()
4225 hs_ep->desc_list = NULL; in dwc2_hsotg_ep_enable()
4237 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_disable() local
4238 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_disable()
4239 int dir_in = hs_ep->dir_in; in dwc2_hsotg_ep_disable()
4240 int index = hs_ep->index; in dwc2_hsotg_ep_disable()
4261 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); in dwc2_hsotg_ep_disable()
4271 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0); in dwc2_hsotg_ep_disable()
4274 kill_all_requests(hsotg, hs_ep, -ESHUTDOWN); in dwc2_hsotg_ep_disable()
4276 hsotg->fifo_map &= ~(1 << hs_ep->fifo_index); in dwc2_hsotg_ep_disable()
4277 hs_ep->fifo_index = 0; in dwc2_hsotg_ep_disable()
4278 hs_ep->fifo_size = 0; in dwc2_hsotg_ep_disable()
4285 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_disable_lock() local
4286 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_disable_lock()
4321 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_dequeue() local
4322 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_dequeue()
4329 if (!on_list(hs_ep, hs_req)) { in dwc2_hsotg_ep_dequeue()
4335 if (req == &hs_ep->req->req) in dwc2_hsotg_ep_dequeue()
4336 dwc2_hsotg_ep_stop_xfr(hs, hs_ep); in dwc2_hsotg_ep_dequeue()
4338 dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET); in dwc2_hsotg_ep_dequeue()
4351 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_gadget_ep_set_wedge() local
4352 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_gadget_ep_set_wedge()
4358 hs_ep->wedged = 1; in dwc2_gadget_ep_set_wedge()
4377 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_sethalt() local
4378 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_sethalt()
4379 int index = hs_ep->index; in dwc2_hsotg_ep_sethalt()
4395 if (hs_ep->isochronous) { in dwc2_hsotg_ep_sethalt()
4400 if (!now && value && !list_empty(&hs_ep->queue)) { in dwc2_hsotg_ep_sethalt()
4406 if (hs_ep->dir_in) { in dwc2_hsotg_ep_sethalt()
4416 hs_ep->wedged = 0; in dwc2_hsotg_ep_sethalt()
4436 hs_ep->wedged = 0; in dwc2_hsotg_ep_sethalt()
4445 hs_ep->halted = value; in dwc2_hsotg_ep_sethalt()
4456 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_sethalt_lock() local
4457 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_sethalt_lock()
4789 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_initep() argument
4802 hs_ep->dir_in = dir_in; in dwc2_hsotg_initep()
4803 hs_ep->index = epnum; in dwc2_hsotg_initep()
4805 snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir); in dwc2_hsotg_initep()
4807 INIT_LIST_HEAD(&hs_ep->queue); in dwc2_hsotg_initep()
4808 INIT_LIST_HEAD(&hs_ep->ep.ep_list); in dwc2_hsotg_initep()
4812 list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list); in dwc2_hsotg_initep()
4814 hs_ep->parent = hsotg; in dwc2_hsotg_initep()
4815 hs_ep->ep.name = hs_ep->name; in dwc2_hsotg_initep()
4818 usb_ep_set_maxpacket_limit(&hs_ep->ep, 8); in dwc2_hsotg_initep()
4820 usb_ep_set_maxpacket_limit(&hs_ep->ep, in dwc2_hsotg_initep()
4822 hs_ep->ep.ops = &dwc2_hsotg_ep_ops; in dwc2_hsotg_initep()
4825 hs_ep->ep.caps.type_control = true; in dwc2_hsotg_initep()
4828 hs_ep->ep.caps.type_iso = true; in dwc2_hsotg_initep()
4829 hs_ep->ep.caps.type_bulk = true; in dwc2_hsotg_initep()
4831 hs_ep->ep.caps.type_int = true; in dwc2_hsotg_initep()
4835 hs_ep->ep.caps.dir_in = true; in dwc2_hsotg_initep()
4837 hs_ep->ep.caps.dir_out = true; in dwc2_hsotg_initep()