Lines Matching +full:usb2 +full:- +full:lpm +full:- +full:disable
1 // SPDX-License-Identifier: GPL-2.0
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
20 #include <linux/dma-mapping.h>
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
39 * success or -EINVAL if wrong Test Selector is passed.
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_test_mode()
57 return -EINVAL; in dwc3_gadget_set_test_mode()
66 * dwc3_gadget_get_link_state - gets current state of usb link
70 * return the link state on success (>= 0) or -ETIMEDOUT.
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_get_link_state()
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
87 * return 0 on success or -ETIMEDOUT.
99 while (--retries) { in dwc3_gadget_set_link_state()
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
108 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_set_link_state()
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_set_link_state()
130 while (--retries) { in dwc3_gadget_set_link_state()
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_set_link_state()
139 return -ETIMEDOUT; in dwc3_gadget_set_link_state()
143 * dwc3_ep_inc_trb - increment a trb index.
153 if (*index == (DWC3_TRB_NUM - 1)) in dwc3_ep_inc_trb()
158 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
163 dwc3_ep_inc_trb(&dep->trb_enqueue); in dwc3_ep_inc_enq()
167 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
172 dwc3_ep_inc_trb(&dep->trb_dequeue); in dwc3_ep_inc_deq()
178 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_del_and_unmap_request()
180 list_del(&req->list); in dwc3_gadget_del_and_unmap_request()
181 req->remaining = 0; in dwc3_gadget_del_and_unmap_request()
182 req->needs_extra_trb = false; in dwc3_gadget_del_and_unmap_request()
184 if (req->request.status == -EINPROGRESS) in dwc3_gadget_del_and_unmap_request()
185 req->request.status = status; in dwc3_gadget_del_and_unmap_request()
187 if (req->trb) in dwc3_gadget_del_and_unmap_request()
188 usb_gadget_unmap_request_by_dev(dwc->sysdev, in dwc3_gadget_del_and_unmap_request()
189 &req->request, req->direction); in dwc3_gadget_del_and_unmap_request()
191 req->trb = NULL; in dwc3_gadget_del_and_unmap_request()
194 if (dep->number > 1) in dwc3_gadget_del_and_unmap_request()
195 pm_runtime_put(dwc->dev); in dwc3_gadget_del_and_unmap_request()
199 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
205 * function will unmap @req and call its ->complete() callback to notify upper
211 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_giveback()
214 req->status = DWC3_REQUEST_STATUS_COMPLETED; in dwc3_gadget_giveback()
216 spin_unlock(&dwc->lock); in dwc3_gadget_giveback()
217 usb_gadget_giveback_request(&dep->endpoint, &req->request); in dwc3_gadget_giveback()
218 spin_lock(&dwc->lock); in dwc3_gadget_giveback()
222 * dwc3_send_gadget_generic_command - issue a generic command for the controller
238 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); in dwc3_send_gadget_generic_command()
239 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); in dwc3_send_gadget_generic_command()
242 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); in dwc3_send_gadget_generic_command()
246 ret = -EINVAL; in dwc3_send_gadget_generic_command()
249 } while (--timeout); in dwc3_send_gadget_generic_command()
252 ret = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
253 status = -ETIMEDOUT; in dwc3_send_gadget_generic_command()
264 * dwc3_send_gadget_ep_cmd - issue an endpoint command
275 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in dwc3_send_gadget_ep_cmd()
276 struct dwc3 *dwc = dep->dwc; in dwc3_send_gadget_ep_cmd()
282 int ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
294 if (dwc->gadget->speed <= USB_SPEED_HIGH) { in dwc3_send_gadget_ep_cmd()
295 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
307 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
318 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", in dwc3_send_gadget_ep_cmd()
323 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); in dwc3_send_gadget_ep_cmd()
324 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); in dwc3_send_gadget_ep_cmd()
325 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); in dwc3_send_gadget_ep_cmd()
348 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); in dwc3_send_gadget_ep_cmd()
350 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); in dwc3_send_gadget_ep_cmd()
359 dev_WARN(dwc->dev, "No resource for %s\n", in dwc3_send_gadget_ep_cmd()
360 dep->name); in dwc3_send_gadget_ep_cmd()
361 ret = -EINVAL; in dwc3_send_gadget_ep_cmd()
371 * Instead of always returning -EINVAL, let's in dwc3_send_gadget_ep_cmd()
373 * the case by returning -EAGAIN. in dwc3_send_gadget_ep_cmd()
375 ret = -EAGAIN; in dwc3_send_gadget_ep_cmd()
378 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); in dwc3_send_gadget_ep_cmd()
383 } while (--timeout); in dwc3_send_gadget_ep_cmd()
386 ret = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
387 cmd_status = -ETIMEDOUT; in dwc3_send_gadget_ep_cmd()
394 dep->flags |= DWC3_EP_TRANSFER_STARTED; in dwc3_send_gadget_ep_cmd()
396 if (ret != -ETIMEDOUT) in dwc3_send_gadget_ep_cmd()
401 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); in dwc3_send_gadget_ep_cmd()
403 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); in dwc3_send_gadget_ep_cmd()
411 struct dwc3 *dwc = dep->dwc; in dwc3_send_clear_stall_ep_cmd()
419 * some (non-compliant) hosts may not send ACK TPs for pending in dwc3_send_clear_stall_ep_cmd()
423 if (dep->direction && in dwc3_send_clear_stall_ep_cmd()
425 (dwc->gadget->speed >= USB_SPEED_SUPER)) in dwc3_send_clear_stall_ep_cmd()
436 u32 offset = (char *) trb - (char *) dep->trb_pool; in dwc3_trb_dma_offset()
438 return dep->trb_pool_dma + offset; in dwc3_trb_dma_offset()
443 struct dwc3 *dwc = dep->dwc; in dwc3_alloc_trb_pool()
445 if (dep->trb_pool) in dwc3_alloc_trb_pool()
448 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, in dwc3_alloc_trb_pool()
450 &dep->trb_pool_dma, GFP_KERNEL); in dwc3_alloc_trb_pool()
451 if (!dep->trb_pool) { in dwc3_alloc_trb_pool()
452 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", in dwc3_alloc_trb_pool()
453 dep->name); in dwc3_alloc_trb_pool()
454 return -ENOMEM; in dwc3_alloc_trb_pool()
462 struct dwc3 *dwc = dep->dwc; in dwc3_free_trb_pool()
464 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, in dwc3_free_trb_pool()
465 dep->trb_pool, dep->trb_pool_dma); in dwc3_free_trb_pool()
467 dep->trb_pool = NULL; in dwc3_free_trb_pool()
468 dep->trb_pool_dma = 0; in dwc3_free_trb_pool()
484 * dwc3_gadget_start_config - configure ep resources
513 * triggered only when called for EP0-out, which always happens first, and which
524 if (dep->number) in dwc3_gadget_start_config()
529 dwc = dep->dwc; in dwc3_gadget_start_config()
536 struct dwc3_ep *dep = dwc->eps[i]; in dwc3_gadget_start_config()
554 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_set_ep_config()
556 comp_desc = dep->endpoint.comp_desc; in dwc3_gadget_set_ep_config()
557 desc = dep->endpoint.desc; in dwc3_gadget_set_ep_config()
565 if (dwc->gadget->speed >= USB_SPEED_SUPER) { in dwc3_gadget_set_ep_config()
566 u32 burst = dep->endpoint.maxburst; in dwc3_gadget_set_ep_config()
568 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); in dwc3_gadget_set_ep_config()
573 params.param2 |= dep->saved_state; in dwc3_gadget_set_ep_config()
578 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) in dwc3_gadget_set_ep_config()
585 dep->stream_capable = true; in dwc3_gadget_set_ep_config()
597 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); in dwc3_gadget_set_ep_config()
603 if (dep->direction) in dwc3_gadget_set_ep_config()
604 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); in dwc3_gadget_set_ep_config()
606 if (desc->bInterval) { in dwc3_gadget_set_ep_config()
617 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); in dwc3_gadget_set_ep_config()
620 dwc->gadget->speed == USB_SPEED_FULL) in dwc3_gadget_set_ep_config()
621 dep->interval = desc->bInterval; in dwc3_gadget_set_ep_config()
623 dep->interval = 1 << (desc->bInterval - 1); in dwc3_gadget_set_ep_config()
635 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
670 * dwc3_gadget_clear_tx_fifo_size - Clears txfifo allocation
683 if (!dwc->do_fifo_resize) in dwc3_gadget_clear_tx_fifos()
687 dep = dwc->eps[1]; in dwc3_gadget_clear_tx_fifos()
688 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_clear_tx_fifos()
694 dwc->last_fifo_depth = fifo_depth; in dwc3_gadget_clear_tx_fifos()
696 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); in dwc3_gadget_clear_tx_fifos()
698 dep = dwc->eps[num]; in dwc3_gadget_clear_tx_fifos()
701 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & in dwc3_gadget_clear_tx_fifos()
704 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); in dwc3_gadget_clear_tx_fifos()
706 dwc->num_ep_resized = 0; in dwc3_gadget_clear_tx_fifos()
710 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
718 * on the configured size for RAM1 - which contains TxFifo -,
732 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_resize_tx_fifos()
743 if (!dwc->do_fifo_resize) in dwc3_gadget_resize_tx_fifos()
747 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) in dwc3_gadget_resize_tx_fifos()
750 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_resize_tx_fifos()
752 if ((dep->endpoint.maxburst > 1 && in dwc3_gadget_resize_tx_fifos()
753 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || in dwc3_gadget_resize_tx_fifos()
754 usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_resize_tx_fifos()
757 if (dep->endpoint.maxburst > 6 && in dwc3_gadget_resize_tx_fifos()
758 usb_endpoint_xfer_bulk(dep->endpoint.desc) && DWC3_IP_IS(DWC31)) in dwc3_gadget_resize_tx_fifos()
759 num_fifos = dwc->tx_fifo_resize_max_num; in dwc3_gadget_resize_tx_fifos()
765 num_in_ep = dwc->max_cfg_eps; in dwc3_gadget_resize_tx_fifos()
766 num_in_ep -= dwc->num_ep_resized; in dwc3_gadget_resize_tx_fifos()
770 remaining = ram1_depth - min_depth - dwc->last_fifo_depth; in dwc3_gadget_resize_tx_fifos()
777 fifo_size = (num_fifos - 1) * fifo; in dwc3_gadget_resize_tx_fifos()
785 /* Check if TXFIFOs start at non-zero addr */ in dwc3_gadget_resize_tx_fifos()
786 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); in dwc3_gadget_resize_tx_fifos()
789 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); in dwc3_gadget_resize_tx_fifos()
791 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
793 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); in dwc3_gadget_resize_tx_fifos()
796 if (dwc->last_fifo_depth >= ram1_depth) { in dwc3_gadget_resize_tx_fifos()
797 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", in dwc3_gadget_resize_tx_fifos()
798 dwc->last_fifo_depth, ram1_depth, in dwc3_gadget_resize_tx_fifos()
799 dep->endpoint.name, fifo_size); in dwc3_gadget_resize_tx_fifos()
805 dwc->last_fifo_depth -= fifo_size; in dwc3_gadget_resize_tx_fifos()
806 return -ENOMEM; in dwc3_gadget_resize_tx_fifos()
809 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); in dwc3_gadget_resize_tx_fifos()
810 dwc->num_ep_resized++; in dwc3_gadget_resize_tx_fifos()
816 * __dwc3_gadget_ep_enable - initializes a hw endpoint
825 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_ep_enable()
826 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_enable()
831 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
845 if (!(dep->flags & DWC3_EP_ENABLED)) { in __dwc3_gadget_ep_enable()
849 dep->type = usb_endpoint_type(desc); in __dwc3_gadget_ep_enable()
850 dep->flags |= DWC3_EP_ENABLED; in __dwc3_gadget_ep_enable()
852 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_enable()
853 reg |= DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_enable()
854 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_enable()
860 dep->trb_dequeue = 0; in __dwc3_gadget_ep_enable()
861 dep->trb_enqueue = 0; in __dwc3_gadget_ep_enable()
862 memset(dep->trb_pool, 0, in __dwc3_gadget_ep_enable()
866 trb_st_hw = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
868 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; in __dwc3_gadget_ep_enable()
869 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
870 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); in __dwc3_gadget_ep_enable()
871 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; in __dwc3_gadget_ep_enable()
872 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_enable()
876 * Issue StartTransfer here with no-op TRB so we can always rely on No in __dwc3_gadget_ep_enable()
887 trb = &dep->trb_pool[0]; in __dwc3_gadget_ep_enable()
899 if (dep->stream_capable) { in __dwc3_gadget_ep_enable()
906 * no-op TRB as normal, but end it immediately. As a in __dwc3_gadget_ep_enable()
924 if (!dep->direction || in __dwc3_gadget_ep_enable()
925 !(dwc->hwparams.hwparams9 & in __dwc3_gadget_ep_enable()
927 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; in __dwc3_gadget_ep_enable()
943 /* - giveback all requests to gadget driver */ in dwc3_remove_requests()
944 while (!list_empty(&dep->started_list)) { in dwc3_remove_requests()
945 req = next_request(&dep->started_list); in dwc3_remove_requests()
947 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_remove_requests()
950 while (!list_empty(&dep->pending_list)) { in dwc3_remove_requests()
951 req = next_request(&dep->pending_list); in dwc3_remove_requests()
953 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_remove_requests()
956 while (!list_empty(&dep->cancelled_list)) { in dwc3_remove_requests()
957 req = next_request(&dep->cancelled_list); in dwc3_remove_requests()
959 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_remove_requests()
964 * __dwc3_gadget_ep_disable - disables a hw endpoint
965 * @dep: the endpoint to disable
975 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_disable()
981 if (dep->flags & DWC3_EP_STALL) in __dwc3_gadget_ep_disable()
984 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); in __dwc3_gadget_ep_disable()
985 reg &= ~DWC3_DALEPENA_EP(dep->number); in __dwc3_gadget_ep_disable()
986 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); in __dwc3_gadget_ep_disable()
988 /* Clear out the ep descriptors for non-ep0 */ in __dwc3_gadget_ep_disable()
989 if (dep->number > 1) { in __dwc3_gadget_ep_disable()
990 dep->endpoint.comp_desc = NULL; in __dwc3_gadget_ep_disable()
991 dep->endpoint.desc = NULL; in __dwc3_gadget_ep_disable()
996 dep->stream_capable = false; in __dwc3_gadget_ep_disable()
997 dep->type = 0; in __dwc3_gadget_ep_disable()
998 dep->flags = 0; in __dwc3_gadget_ep_disable()
1003 /* -------------------------------------------------------------------------- */
1008 return -EINVAL; in dwc3_gadget_ep0_enable()
1013 return -EINVAL; in dwc3_gadget_ep0_disable()
1016 /* -------------------------------------------------------------------------- */
1026 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { in dwc3_gadget_ep_enable()
1028 return -EINVAL; in dwc3_gadget_ep_enable()
1031 if (!desc->wMaxPacketSize) { in dwc3_gadget_ep_enable()
1033 return -EINVAL; in dwc3_gadget_ep_enable()
1037 dwc = dep->dwc; in dwc3_gadget_ep_enable()
1039 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, in dwc3_gadget_ep_enable()
1041 dep->name)) in dwc3_gadget_ep_enable()
1044 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1046 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_enable()
1060 return -EINVAL; in dwc3_gadget_ep_disable()
1064 dwc = dep->dwc; in dwc3_gadget_ep_disable()
1066 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), in dwc3_gadget_ep_disable()
1068 dep->name)) in dwc3_gadget_ep_disable()
1071 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1073 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_disable()
1088 req->direction = dep->direction; in dwc3_gadget_ep_alloc_request()
1089 req->epnum = dep->number; in dwc3_gadget_ep_alloc_request()
1090 req->dep = dep; in dwc3_gadget_ep_alloc_request()
1091 req->status = DWC3_REQUEST_STATUS_UNKNOWN; in dwc3_gadget_ep_alloc_request()
1095 return &req->request; in dwc3_gadget_ep_alloc_request()
1108 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1121 tmp = DWC3_TRB_NUM - 1; in dwc3_ep_prev_trb()
1123 return &dep->trb_pool[tmp - 1]; in dwc3_ep_prev_trb()
1132 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs in dwc3_calc_trbs_left()
1135 if (dep->trb_enqueue == dep->trb_dequeue) { in dwc3_calc_trbs_left()
1140 if (!list_empty(&dep->started_list)) in dwc3_calc_trbs_left()
1143 return DWC3_TRB_NUM - 1; in dwc3_calc_trbs_left()
1146 trbs_left = dep->trb_dequeue - dep->trb_enqueue; in dwc3_calc_trbs_left()
1147 trbs_left &= (DWC3_TRB_NUM - 1); in dwc3_calc_trbs_left()
1149 if (dep->trb_dequeue < dep->trb_enqueue) in dwc3_calc_trbs_left()
1150 trbs_left--; in dwc3_calc_trbs_left()
1161 struct dwc3 *dwc = dep->dwc; in __dwc3_prepare_one_trb()
1162 struct usb_gadget *gadget = dwc->gadget; in __dwc3_prepare_one_trb()
1163 enum usb_device_speed speed = gadget->speed; in __dwc3_prepare_one_trb()
1165 trb->size = DWC3_TRB_SIZE_LENGTH(length); in __dwc3_prepare_one_trb()
1166 trb->bpl = lower_32_bits(dma); in __dwc3_prepare_one_trb()
1167 trb->bph = upper_32_bits(dma); in __dwc3_prepare_one_trb()
1169 switch (usb_endpoint_type(dep->endpoint.desc)) { in __dwc3_prepare_one_trb()
1171 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; in __dwc3_prepare_one_trb()
1176 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; in __dwc3_prepare_one_trb()
1192 * - DATA0 in __dwc3_prepare_one_trb()
1195 * - DATA1, DATA0 in __dwc3_prepare_one_trb()
1198 * - DATA2, DATA1, DATA0 in __dwc3_prepare_one_trb()
1201 struct usb_ep *ep = &dep->endpoint; in __dwc3_prepare_one_trb()
1203 unsigned int maxp = usb_endpoint_maxp(ep->desc); in __dwc3_prepare_one_trb()
1206 mult--; in __dwc3_prepare_one_trb()
1209 mult--; in __dwc3_prepare_one_trb()
1211 trb->size |= DWC3_TRB_SIZE_PCM1(mult); in __dwc3_prepare_one_trb()
1214 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; in __dwc3_prepare_one_trb()
1218 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in __dwc3_prepare_one_trb()
1223 trb->ctrl = DWC3_TRBCTL_NORMAL; in __dwc3_prepare_one_trb()
1230 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", in __dwc3_prepare_one_trb()
1231 usb_endpoint_type(dep->endpoint.desc)); in __dwc3_prepare_one_trb()
1238 if (usb_endpoint_dir_out(dep->endpoint.desc)) { in __dwc3_prepare_one_trb()
1239 if (!dep->stream_capable) in __dwc3_prepare_one_trb()
1240 trb->ctrl |= DWC3_TRB_CTRL_CSP; in __dwc3_prepare_one_trb()
1243 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; in __dwc3_prepare_one_trb()
1247 trb->ctrl |= DWC3_TRB_CTRL_IOC; in __dwc3_prepare_one_trb()
1250 trb->ctrl |= DWC3_TRB_CTRL_CHN; in __dwc3_prepare_one_trb()
1251 else if (dep->stream_capable && is_last) in __dwc3_prepare_one_trb()
1252 trb->ctrl |= DWC3_TRB_CTRL_LST; in __dwc3_prepare_one_trb()
1254 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) in __dwc3_prepare_one_trb()
1255 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); in __dwc3_prepare_one_trb()
1257 trb->ctrl |= DWC3_TRB_CTRL_HWO; in __dwc3_prepare_one_trb()
1265 * dwc3_prepare_one_trb - setup one TRB from one request
1281 unsigned int stream_id = req->request.stream_id; in dwc3_prepare_one_trb()
1282 unsigned int short_not_ok = req->request.short_not_ok; in dwc3_prepare_one_trb()
1283 unsigned int no_interrupt = req->request.no_interrupt; in dwc3_prepare_one_trb()
1284 unsigned int is_last = req->request.is_last; in dwc3_prepare_one_trb()
1287 dma = dep->dwc->bounce_addr; in dwc3_prepare_one_trb()
1288 else if (req->request.num_sgs > 0) in dwc3_prepare_one_trb()
1289 dma = sg_dma_address(req->start_sg); in dwc3_prepare_one_trb()
1291 dma = req->request.dma; in dwc3_prepare_one_trb()
1293 trb = &dep->trb_pool[dep->trb_enqueue]; in dwc3_prepare_one_trb()
1295 if (!req->trb) { in dwc3_prepare_one_trb()
1297 req->trb = trb; in dwc3_prepare_one_trb()
1298 req->trb_dma = dwc3_trb_dma_offset(dep, trb); in dwc3_prepare_one_trb()
1301 req->num_trbs++; in dwc3_prepare_one_trb()
1310 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_needs_extra_trb()
1311 unsigned int rem = req->request.length % maxp; in dwc3_needs_extra_trb()
1313 if ((req->request.length && req->request.zero && !rem && in dwc3_needs_extra_trb()
1314 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || in dwc3_needs_extra_trb()
1315 (!req->direction && rem)) in dwc3_needs_extra_trb()
1322 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1334 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); in dwc3_prepare_last_sg()
1335 unsigned int rem = req->request.length % maxp; in dwc3_prepare_last_sg()
1344 req->needs_extra_trb = num_trbs > 1; in dwc3_prepare_last_sg()
1347 if (req->direction || req->request.length) in dwc3_prepare_last_sg()
1349 req->needs_extra_trb, node, false, false); in dwc3_prepare_last_sg()
1352 if ((!req->direction && !req->request.length) || req->needs_extra_trb) in dwc3_prepare_last_sg()
1354 req->direction ? 0 : maxp - rem, in dwc3_prepare_last_sg()
1363 struct scatterlist *sg = req->start_sg; in dwc3_prepare_trbs_sg()
1366 unsigned int length = req->request.length; in dwc3_prepare_trbs_sg()
1367 unsigned int remaining = req->request.num_mapped_sgs in dwc3_prepare_trbs_sg()
1368 - req->num_queued_sgs; in dwc3_prepare_trbs_sg()
1369 unsigned int num_trbs = req->num_trbs; in dwc3_prepare_trbs_sg()
1376 for_each_sg(req->request.sg, s, req->num_queued_sgs, i) in dwc3_prepare_trbs_sg()
1377 length -= sg_dma_len(s); in dwc3_prepare_trbs_sg()
1387 length -= trb_length; in dwc3_prepare_trbs_sg()
1396 if ((i == remaining - 1) || !length) in dwc3_prepare_trbs_sg()
1429 req->start_sg = sg_next(s); in dwc3_prepare_trbs_sg()
1431 req->num_queued_sgs++; in dwc3_prepare_trbs_sg()
1432 req->num_pending_sgs--; in dwc3_prepare_trbs_sg()
1440 req->num_pending_sgs = 0; in dwc3_prepare_trbs_sg()
1448 return req->num_trbs - num_trbs; in dwc3_prepare_trbs_sg()
1454 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); in dwc3_prepare_trbs_linear()
1458 * dwc3_prepare_trbs - setup TRBs from requests
1484 list_for_each_entry(req, &dep->started_list, list) { in dwc3_prepare_trbs()
1485 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1487 if (!ret || req->num_pending_sgs) in dwc3_prepare_trbs()
1499 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1503 list_for_each_entry_safe(req, n, &dep->pending_list, list) { in dwc3_prepare_trbs()
1504 struct dwc3 *dwc = dep->dwc; in dwc3_prepare_trbs()
1506 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, in dwc3_prepare_trbs()
1507 dep->direction); in dwc3_prepare_trbs()
1511 req->sg = req->request.sg; in dwc3_prepare_trbs()
1512 req->start_sg = req->sg; in dwc3_prepare_trbs()
1513 req->num_queued_sgs = 0; in dwc3_prepare_trbs()
1514 req->num_pending_sgs = req->request.num_mapped_sgs; in dwc3_prepare_trbs()
1516 if (req->num_pending_sgs > 0) { in dwc3_prepare_trbs()
1518 if (req->num_pending_sgs) in dwc3_prepare_trbs()
1532 if (dep->stream_capable && req->request.is_last) in dwc3_prepare_trbs()
1558 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); in __dwc3_gadget_kick_transfer()
1567 req = next_request(&dep->started_list); in __dwc3_gadget_kick_transfer()
1569 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_kick_transfer()
1576 params.param0 = upper_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1577 params.param1 = lower_32_bits(req->trb_dma); in __dwc3_gadget_kick_transfer()
1580 if (dep->stream_capable) in __dwc3_gadget_kick_transfer()
1581 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); in __dwc3_gadget_kick_transfer()
1583 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_kick_transfer()
1584 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); in __dwc3_gadget_kick_transfer()
1587 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_kick_transfer()
1594 if (ret == -EAGAIN) in __dwc3_gadget_kick_transfer()
1599 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_kick_transfer()
1603 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in __dwc3_gadget_kick_transfer()
1609 if (dep->stream_capable && req->request.is_last) in __dwc3_gadget_kick_transfer()
1610 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; in __dwc3_gadget_kick_transfer()
1619 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_get_frame()
1624 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1627 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1631 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1632 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1636 * internal 16-bit microframe, the START TRANSFER command will pass and the
1639 * other conditions, the START TRANSFER command will fail with bus-expiry.
1644 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1647 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1671 while (dep->combo_num < 2) { in dwc3_gadget_start_isoc_quirk()
1678 * 4 uframes in the future with BIT[15:14] as dep->combo_num in dwc3_gadget_start_isoc_quirk()
1680 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1681 test_frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1682 test_frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1684 params.param0 = upper_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1685 params.param1 = lower_32_bits(dep->dwc->bounce_addr); in dwc3_gadget_start_isoc_quirk()
1691 /* Redo if some other failure beside bus-expiry is received */ in dwc3_gadget_start_isoc_quirk()
1692 if (cmd_status && cmd_status != -EAGAIN) { in dwc3_gadget_start_isoc_quirk()
1693 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1694 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1699 if (dep->combo_num == 0) in dwc3_gadget_start_isoc_quirk()
1700 dep->start_cmd_status = cmd_status; in dwc3_gadget_start_isoc_quirk()
1702 dep->combo_num++; in dwc3_gadget_start_isoc_quirk()
1715 test0 = (dep->start_cmd_status == 0); in dwc3_gadget_start_isoc_quirk()
1719 dep->combo_num = 1; in dwc3_gadget_start_isoc_quirk()
1721 dep->combo_num = 2; in dwc3_gadget_start_isoc_quirk()
1723 dep->combo_num = 3; in dwc3_gadget_start_isoc_quirk()
1725 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1727 dep->frame_number &= DWC3_FRNUMBER_MASK; in dwc3_gadget_start_isoc_quirk()
1728 dep->frame_number |= dep->combo_num << 14; in dwc3_gadget_start_isoc_quirk()
1729 dep->frame_number += max_t(u32, 4, dep->interval); in dwc3_gadget_start_isoc_quirk()
1732 dep->start_cmd_status = 0; in dwc3_gadget_start_isoc_quirk()
1733 dep->combo_num = 0; in dwc3_gadget_start_isoc_quirk()
1740 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; in __dwc3_gadget_start_isoc()
1741 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_start_isoc()
1745 if (list_empty(&dep->pending_list) && in __dwc3_gadget_start_isoc()
1746 list_empty(&dep->started_list)) { in __dwc3_gadget_start_isoc()
1747 dep->flags |= DWC3_EP_PENDING_REQUEST; in __dwc3_gadget_start_isoc()
1748 return -EAGAIN; in __dwc3_gadget_start_isoc()
1751 if (!dwc->dis_start_transfer_quirk && in __dwc3_gadget_start_isoc()
1754 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) in __dwc3_gadget_start_isoc()
1758 if (desc->bInterval <= 14 && in __dwc3_gadget_start_isoc()
1759 dwc->gadget->speed >= USB_SPEED_HIGH) { in __dwc3_gadget_start_isoc()
1762 (dep->frame_number & DWC3_FRNUMBER_MASK); in __dwc3_gadget_start_isoc()
1773 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | in __dwc3_gadget_start_isoc()
1776 dep->frame_number += BIT(14); in __dwc3_gadget_start_isoc()
1780 dep->frame_number = DWC3_ALIGN_FRAME(dep, i + 1); in __dwc3_gadget_start_isoc()
1783 if (ret != -EAGAIN) in __dwc3_gadget_start_isoc()
1788 * After a number of unsuccessful start attempts due to bus-expiry in __dwc3_gadget_start_isoc()
1792 if (ret == -EAGAIN) { in __dwc3_gadget_start_isoc()
1798 DWC3_DEPCMD_PARAM(dep->resource_index); in __dwc3_gadget_start_isoc()
1800 dep->resource_index = 0; in __dwc3_gadget_start_isoc()
1805 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in __dwc3_gadget_start_isoc()
1813 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_queue()
1815 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { in __dwc3_gadget_ep_queue()
1816 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", in __dwc3_gadget_ep_queue()
1817 dep->name); in __dwc3_gadget_ep_queue()
1818 return -ESHUTDOWN; in __dwc3_gadget_ep_queue()
1821 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", in __dwc3_gadget_ep_queue()
1822 &req->request, req->dep->name)) in __dwc3_gadget_ep_queue()
1823 return -EINVAL; in __dwc3_gadget_ep_queue()
1825 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, in __dwc3_gadget_ep_queue()
1827 dep->name, &req->request)) in __dwc3_gadget_ep_queue()
1828 return -EINVAL; in __dwc3_gadget_ep_queue()
1830 pm_runtime_get(dwc->dev); in __dwc3_gadget_ep_queue()
1832 req->request.actual = 0; in __dwc3_gadget_ep_queue()
1833 req->request.status = -EINPROGRESS; in __dwc3_gadget_ep_queue()
1837 list_add_tail(&req->list, &dep->pending_list); in __dwc3_gadget_ep_queue()
1838 req->status = DWC3_REQUEST_STATUS_QUEUED; in __dwc3_gadget_ep_queue()
1840 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) in __dwc3_gadget_ep_queue()
1847 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || in __dwc3_gadget_ep_queue()
1848 (dep->flags & DWC3_EP_WEDGE) || in __dwc3_gadget_ep_queue()
1849 (dep->flags & DWC3_EP_STALL)) { in __dwc3_gadget_ep_queue()
1850 dep->flags |= DWC3_EP_DELAY_START; in __dwc3_gadget_ep_queue()
1857 * (micro-)frame number. in __dwc3_gadget_ep_queue()
1862 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_queue()
1863 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) && in __dwc3_gadget_ep_queue()
1864 !(dep->flags & DWC3_EP_TRANSFER_STARTED)) in __dwc3_gadget_ep_queue()
1867 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { in __dwc3_gadget_ep_queue()
1868 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) in __dwc3_gadget_ep_queue()
1883 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_queue()
1889 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_queue()
1891 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_queue()
1900 /* If req->trb is not set, then the request has not started */ in dwc3_gadget_ep_skip_trbs()
1901 if (!req->trb) in dwc3_gadget_ep_skip_trbs()
1914 for (i = 0; i < req->num_trbs; i++) { in dwc3_gadget_ep_skip_trbs()
1917 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_skip_trbs()
1918 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_skip_trbs()
1922 req->num_trbs = 0; in dwc3_gadget_ep_skip_trbs()
1929 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_cleanup_cancelled_requests()
1931 list_for_each_entry_safe(req, tmp, &dep->cancelled_list, list) { in dwc3_gadget_ep_cleanup_cancelled_requests()
1933 switch (req->status) { in dwc3_gadget_ep_cleanup_cancelled_requests()
1935 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); in dwc3_gadget_ep_cleanup_cancelled_requests()
1938 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
1941 dwc3_gadget_giveback(dep, req, -EPIPE); in dwc3_gadget_ep_cleanup_cancelled_requests()
1944 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); in dwc3_gadget_ep_cleanup_cancelled_requests()
1945 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_cleanup_cancelled_requests()
1958 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_dequeue()
1965 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
1967 list_for_each_entry(r, &dep->cancelled_list, list) { in dwc3_gadget_ep_dequeue()
1972 list_for_each_entry(r, &dep->pending_list, list) { in dwc3_gadget_ep_dequeue()
1974 dwc3_gadget_giveback(dep, req, -ECONNRESET); in dwc3_gadget_ep_dequeue()
1979 list_for_each_entry(r, &dep->started_list, list) { in dwc3_gadget_ep_dequeue()
1990 list_for_each_entry_safe(r, t, &dep->started_list, list) in dwc3_gadget_ep_dequeue()
1994 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_ep_dequeue()
2000 dev_err(dwc->dev, "request %pK was not queued to %s\n", in dwc3_gadget_ep_dequeue()
2001 request, ep->name); in dwc3_gadget_ep_dequeue()
2002 ret = -EINVAL; in dwc3_gadget_ep_dequeue()
2004 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_dequeue()
2012 struct dwc3 *dwc = dep->dwc; in __dwc3_gadget_ep_set_halt()
2017 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { in __dwc3_gadget_ep_set_halt()
2018 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); in __dwc3_gadget_ep_set_halt()
2019 return -EINVAL; in __dwc3_gadget_ep_set_halt()
2030 if (dep->number > 1) in __dwc3_gadget_ep_set_halt()
2031 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); in __dwc3_gadget_ep_set_halt()
2033 trb = &dwc->ep0_trb[dep->trb_enqueue]; in __dwc3_gadget_ep_set_halt()
2035 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; in __dwc3_gadget_ep_set_halt()
2036 started = !list_empty(&dep->started_list); in __dwc3_gadget_ep_set_halt()
2038 if (!protocol && ((dep->direction && transfer_in_flight) || in __dwc3_gadget_ep_set_halt()
2039 (!dep->direction && started))) { in __dwc3_gadget_ep_set_halt()
2040 return -EAGAIN; in __dwc3_gadget_ep_set_halt()
2046 dev_err(dwc->dev, "failed to set STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2047 dep->name); in __dwc3_gadget_ep_set_halt()
2049 dep->flags |= DWC3_EP_STALL; in __dwc3_gadget_ep_set_halt()
2056 if (dep->number <= 1) { in __dwc3_gadget_ep_set_halt()
2057 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2063 list_for_each_entry_safe(req, tmp, &dep->started_list, list) in __dwc3_gadget_ep_set_halt()
2066 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) { in __dwc3_gadget_ep_set_halt()
2067 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; in __dwc3_gadget_ep_set_halt()
2075 dev_err(dwc->dev, "failed to clear STALL on %s\n", in __dwc3_gadget_ep_set_halt()
2076 dep->name); in __dwc3_gadget_ep_set_halt()
2080 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in __dwc3_gadget_ep_set_halt()
2082 if ((dep->flags & DWC3_EP_DELAY_START) && in __dwc3_gadget_ep_set_halt()
2083 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in __dwc3_gadget_ep_set_halt()
2086 dep->flags &= ~DWC3_EP_DELAY_START; in __dwc3_gadget_ep_set_halt()
2095 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_halt()
2101 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2103 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_halt()
2111 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_set_wedge()
2115 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2116 dep->flags |= DWC3_EP_WEDGE; in dwc3_gadget_ep_set_wedge()
2118 if (dep->number == 0 || dep->number == 1) in dwc3_gadget_ep_set_wedge()
2122 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_ep_set_wedge()
2127 /* -------------------------------------------------------------------------- */
2137 .disable = dwc3_gadget_ep0_disable,
2148 .disable = dwc3_gadget_ep_disable,
2157 /* -------------------------------------------------------------------------- */
2181 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2194 return -EINVAL; in __dwc3_gadget_wakeup()
2199 dev_err(dwc->dev, "failed to put link in Recovery\n"); in __dwc3_gadget_wakeup()
2206 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in __dwc3_gadget_wakeup()
2208 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in __dwc3_gadget_wakeup()
2214 while (retries--) { in __dwc3_gadget_wakeup()
2215 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in __dwc3_gadget_wakeup()
2223 dev_err(dwc->dev, "failed to send remote wakeup\n"); in __dwc3_gadget_wakeup()
2224 return -EINVAL; in __dwc3_gadget_wakeup()
2236 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_wakeup()
2238 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_wakeup()
2249 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2250 g->is_selfpowered = !!is_selfpowered; in dwc3_gadget_set_selfpowered()
2251 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_selfpowered()
2260 for (epnum = 2; epnum < dwc->num_eps; epnum++) { in dwc3_stop_active_transfers()
2263 dep = dwc->eps[epnum]; in dwc3_stop_active_transfers()
2273 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2277 ssp_rate = dwc->max_ssp_rate; in __dwc3_gadget_set_ssp_rate()
2279 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_ssp_rate()
2285 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) in __dwc3_gadget_set_ssp_rate()
2289 dwc->max_ssp_rate != USB_SSP_GEN_2x1) in __dwc3_gadget_set_ssp_rate()
2292 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_ssp_rate()
2300 speed = dwc->gadget_max_speed; in __dwc3_gadget_set_speed()
2301 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) in __dwc3_gadget_set_speed()
2302 speed = dwc->maximum_speed; in __dwc3_gadget_set_speed()
2310 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_set_speed()
2316 * bit if we try to force the IP to USB2-only mode. in __dwc3_gadget_set_speed()
2327 !dwc->dis_metastability_quirk) { in __dwc3_gadget_set_speed()
2347 dev_err(dwc->dev, "invalid speed (%d)\n", speed); in __dwc3_gadget_set_speed()
2361 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_set_speed()
2369 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_run_stop()
2372 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_run_stop()
2383 if (dwc->has_hibernation) in dwc3_gadget_run_stop()
2387 dwc->pullups_connected = true; in dwc3_gadget_run_stop()
2391 if (dwc->has_hibernation && !suspend) in dwc3_gadget_run_stop()
2394 dwc->pullups_connected = false; in dwc3_gadget_run_stop()
2400 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_run_stop()
2402 } while (--timeout && !(!is_on ^ !reg)); in dwc3_gadget_run_stop()
2405 return -ETIMEDOUT; in dwc3_gadget_run_stop()
2426 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) { in dwc3_gadget_pullup()
2427 reinit_completion(&dwc->ep0_in_setup); in dwc3_gadget_pullup()
2429 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, in dwc3_gadget_pullup()
2432 dev_warn(dwc->dev, "timed out waiting for SETUP phase\n"); in dwc3_gadget_pullup()
2441 pm_runtime_barrier(dwc->dev); in dwc3_gadget_pullup()
2442 if (pm_runtime_suspended(dwc->dev)) in dwc3_gadget_pullup()
2451 ret = pm_runtime_get_sync(dwc->dev); in dwc3_gadget_pullup()
2453 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2458 * Synchronize and disable any further event handling while controller in dwc3_gadget_pullup()
2461 disable_irq(dwc->irq_gadget); in dwc3_gadget_pullup()
2463 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_pullup()
2468 dwc->connected = false; in dwc3_gadget_pullup()
2471 * Section 4.1.8 Table 4-7, it states that for a device-initiated in dwc3_gadget_pullup()
2486 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); in dwc3_gadget_pullup()
2489 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); in dwc3_gadget_pullup()
2490 dwc->ev_buf->lpos = (dwc->ev_buf->lpos + count) % in dwc3_gadget_pullup()
2491 dwc->ev_buf->length; in dwc3_gadget_pullup()
2498 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_pullup()
2499 enable_irq(dwc->irq_gadget); in dwc3_gadget_pullup()
2501 pm_runtime_put(dwc->dev); in dwc3_gadget_pullup()
2522 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ in dwc3_gadget_enable_irq()
2526 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); in dwc3_gadget_enable_irq()
2532 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); in dwc3_gadget_disable_irq()
2539 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2552 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2566 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_setup_nump()
2569 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; in dwc3_gadget_setup_nump()
2573 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_setup_nump()
2576 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_setup_nump()
2586 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if in __dwc3_gadget_start()
2587 * the core supports IMOD, disable it. in __dwc3_gadget_start()
2589 if (dwc->imod_interval) { in __dwc3_gadget_start()
2590 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in __dwc3_gadget_start()
2591 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in __dwc3_gadget_start()
2593 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); in __dwc3_gadget_start()
2603 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); in __dwc3_gadget_start()
2609 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); in __dwc3_gadget_start()
2620 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in __dwc3_gadget_start()
2622 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in __dwc3_gadget_start()
2627 dep = dwc->eps[0]; in __dwc3_gadget_start()
2630 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2634 dep = dwc->eps[1]; in __dwc3_gadget_start()
2637 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in __dwc3_gadget_start()
2642 dwc->ep0state = EP0_SETUP_PHASE; in __dwc3_gadget_start()
2643 dwc->link_state = DWC3_LINK_STATE_SS_DIS; in __dwc3_gadget_start()
2644 dwc->delayed_status = false; in __dwc3_gadget_start()
2652 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_start()
2666 irq = dwc->irq_gadget; in dwc3_gadget_start()
2668 IRQF_SHARED, "dwc3", dwc->ev_buf); in dwc3_gadget_start()
2670 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", in dwc3_gadget_start()
2675 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_start()
2676 dwc->gadget_driver = driver; in dwc3_gadget_start()
2677 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_start()
2685 __dwc3_gadget_ep_disable(dwc->eps[0]); in __dwc3_gadget_stop()
2686 __dwc3_gadget_ep_disable(dwc->eps[1]); in __dwc3_gadget_stop()
2694 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_stop()
2695 dwc->gadget_driver = NULL; in dwc3_gadget_stop()
2696 dwc->max_cfg_eps = 0; in dwc3_gadget_stop()
2697 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_stop()
2699 free_irq(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_stop()
2709 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
2710 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; in dwc3_gadget_config_params()
2713 if (!dwc->dis_enblslpm_quirk) { in dwc3_gadget_config_params()
2723 params->besl_baseline = 1; in dwc3_gadget_config_params()
2724 if (dwc->is_utmi_l1_suspend) in dwc3_gadget_config_params()
2725 params->besl_deep = in dwc3_gadget_config_params()
2726 clamp_t(u8, dwc->hird_threshold, 2, 15); in dwc3_gadget_config_params()
2730 if (dwc->dis_u1_entry_quirk) in dwc3_gadget_config_params()
2731 params->bU1devExitLat = 0; in dwc3_gadget_config_params()
2733 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; in dwc3_gadget_config_params()
2736 if (dwc->dis_u2_entry_quirk) in dwc3_gadget_config_params()
2737 params->bU2DevExitLat = 0; in dwc3_gadget_config_params()
2739 params->bU2DevExitLat = in dwc3_gadget_config_params()
2749 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_speed()
2750 dwc->gadget_max_speed = speed; in dwc3_gadget_set_speed()
2751 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_speed()
2760 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
2761 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_set_ssp_rate()
2762 dwc->gadget_ssp_rate = rate; in dwc3_gadget_set_ssp_rate()
2763 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_set_ssp_rate()
2772 if (dwc->usb2_phy) in dwc3_gadget_vbus_draw()
2773 return usb_phy_set_power(dwc->usb2_phy, mA); in dwc3_gadget_vbus_draw()
2775 if (!dwc->usb_psy) in dwc3_gadget_vbus_draw()
2776 return -EOPNOTSUPP; in dwc3_gadget_vbus_draw()
2779 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); in dwc3_gadget_vbus_draw()
2785 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
2802 if (!dwc->do_fifo_resize) in dwc3_gadget_check_config()
2805 list_for_each_entry(ep, &g->ep_list, ep_list) { in dwc3_gadget_check_config()
2807 if (ep->claimed && (ep->address & USB_DIR_IN)) in dwc3_gadget_check_config()
2811 if (ep_num <= dwc->max_cfg_eps) in dwc3_gadget_check_config()
2815 dwc->max_cfg_eps = ep_num; in dwc3_gadget_check_config()
2817 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); in dwc3_gadget_check_config()
2819 fifo_size += dwc->max_cfg_eps; in dwc3_gadget_check_config()
2822 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); in dwc3_gadget_check_config()
2824 return -ENOMEM; in dwc3_gadget_check_config()
2834 spin_lock_irqsave(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
2835 dwc->async_callbacks = enable; in dwc3_gadget_async_callbacks()
2836 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_gadget_async_callbacks()
2854 /* -------------------------------------------------------------------------- */
2858 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_control_endpoint()
2860 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); in dwc3_gadget_init_control_endpoint()
2861 dep->endpoint.maxburst = 1; in dwc3_gadget_init_control_endpoint()
2862 dep->endpoint.ops = &dwc3_gadget_ep0_ops; in dwc3_gadget_init_control_endpoint()
2863 if (!dep->direction) in dwc3_gadget_init_control_endpoint()
2864 dwc->gadget->ep0 = &dep->endpoint; in dwc3_gadget_init_control_endpoint()
2866 dep->endpoint.caps.type_control = true; in dwc3_gadget_init_control_endpoint()
2873 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_in_endpoint()
2882 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); in dwc3_gadget_init_in_endpoint()
2898 if (dwc->maximum_speed >= USB_SPEED_SUPER) in dwc3_gadget_init_in_endpoint()
2903 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_in_endpoint()
2905 dep->endpoint.max_streams = 16; in dwc3_gadget_init_in_endpoint()
2906 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_in_endpoint()
2907 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_in_endpoint()
2908 &dwc->gadget->ep_list); in dwc3_gadget_init_in_endpoint()
2909 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_in_endpoint()
2910 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_in_endpoint()
2911 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_in_endpoint()
2918 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_init_out_endpoint()
2928 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); in dwc3_gadget_init_out_endpoint()
2945 size -= (3 * 8) + 16; in dwc3_gadget_init_out_endpoint()
2951 usb_ep_set_maxpacket_limit(&dep->endpoint, size); in dwc3_gadget_init_out_endpoint()
2952 dep->endpoint.max_streams = 16; in dwc3_gadget_init_out_endpoint()
2953 dep->endpoint.ops = &dwc3_gadget_ep_ops; in dwc3_gadget_init_out_endpoint()
2954 list_add_tail(&dep->endpoint.ep_list, in dwc3_gadget_init_out_endpoint()
2955 &dwc->gadget->ep_list); in dwc3_gadget_init_out_endpoint()
2956 dep->endpoint.caps.type_iso = true; in dwc3_gadget_init_out_endpoint()
2957 dep->endpoint.caps.type_bulk = true; in dwc3_gadget_init_out_endpoint()
2958 dep->endpoint.caps.type_int = true; in dwc3_gadget_init_out_endpoint()
2972 return -ENOMEM; in dwc3_gadget_init_endpoint()
2974 dep->dwc = dwc; in dwc3_gadget_init_endpoint()
2975 dep->number = epnum; in dwc3_gadget_init_endpoint()
2976 dep->direction = direction; in dwc3_gadget_init_endpoint()
2977 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); in dwc3_gadget_init_endpoint()
2978 dwc->eps[epnum] = dep; in dwc3_gadget_init_endpoint()
2979 dep->combo_num = 0; in dwc3_gadget_init_endpoint()
2980 dep->start_cmd_status = 0; in dwc3_gadget_init_endpoint()
2982 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, in dwc3_gadget_init_endpoint()
2985 dep->endpoint.name = dep->name; in dwc3_gadget_init_endpoint()
2987 if (!(dep->number > 1)) { in dwc3_gadget_init_endpoint()
2988 dep->endpoint.desc = &dwc3_gadget_ep0_desc; in dwc3_gadget_init_endpoint()
2989 dep->endpoint.comp_desc = NULL; in dwc3_gadget_init_endpoint()
3002 dep->endpoint.caps.dir_in = direction; in dwc3_gadget_init_endpoint()
3003 dep->endpoint.caps.dir_out = !direction; in dwc3_gadget_init_endpoint()
3005 INIT_LIST_HEAD(&dep->pending_list); in dwc3_gadget_init_endpoint()
3006 INIT_LIST_HEAD(&dep->started_list); in dwc3_gadget_init_endpoint()
3007 INIT_LIST_HEAD(&dep->cancelled_list); in dwc3_gadget_init_endpoint()
3018 INIT_LIST_HEAD(&dwc->gadget->ep_list); in dwc3_gadget_init_endpoints()
3037 dep = dwc->eps[epnum]; in dwc3_gadget_free_endpoints()
3042 * bi-directional USB endpoint 0. in dwc3_gadget_free_endpoints()
3051 list_del(&dep->endpoint.ep_list); in dwc3_gadget_free_endpoints()
3054 debugfs_remove_recursive(debugfs_lookup(dep->name, in dwc3_gadget_free_endpoints()
3055 debugfs_lookup(dev_name(dep->dwc->dev), in dwc3_gadget_free_endpoints()
3061 /* -------------------------------------------------------------------------- */
3072 req->num_trbs--; in dwc3_gadget_ep_reclaim_completed_trb()
3084 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) in dwc3_gadget_ep_reclaim_completed_trb()
3085 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3089 * have the Isoc-First type. Track and report its interval frame number. in dwc3_gadget_ep_reclaim_completed_trb()
3091 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_ep_reclaim_completed_trb()
3092 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { in dwc3_gadget_ep_reclaim_completed_trb()
3095 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); in dwc3_gadget_ep_reclaim_completed_trb()
3096 frame_number &= ~(dep->interval - 1); in dwc3_gadget_ep_reclaim_completed_trb()
3097 req->request.frame_number = frame_number; in dwc3_gadget_ep_reclaim_completed_trb()
3103 * TRB. Don't add it to req->remaining calculation. in dwc3_gadget_ep_reclaim_completed_trb()
3105 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && in dwc3_gadget_ep_reclaim_completed_trb()
3106 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { in dwc3_gadget_ep_reclaim_completed_trb()
3107 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; in dwc3_gadget_ep_reclaim_completed_trb()
3111 count = trb->size & DWC3_TRB_SIZE_MASK; in dwc3_gadget_ep_reclaim_completed_trb()
3112 req->remaining += count; in dwc3_gadget_ep_reclaim_completed_trb()
3114 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) in dwc3_gadget_ep_reclaim_completed_trb()
3117 if (event->status & DEPEVT_STATUS_SHORT && !chain) in dwc3_gadget_ep_reclaim_completed_trb()
3120 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || in dwc3_gadget_ep_reclaim_completed_trb()
3121 (trb->ctrl & DWC3_TRB_CTRL_LST)) in dwc3_gadget_ep_reclaim_completed_trb()
3131 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3132 struct scatterlist *sg = req->sg; in dwc3_gadget_ep_reclaim_trb_sg()
3134 unsigned int num_queued = req->num_queued_sgs; in dwc3_gadget_ep_reclaim_trb_sg()
3139 trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_sg()
3141 req->sg = sg_next(s); in dwc3_gadget_ep_reclaim_trb_sg()
3142 req->num_queued_sgs--; in dwc3_gadget_ep_reclaim_trb_sg()
3157 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; in dwc3_gadget_ep_reclaim_trb_linear()
3165 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0; in dwc3_gadget_ep_request_completed()
3174 if (req->request.num_mapped_sgs) in dwc3_gadget_ep_cleanup_completed_request()
3181 req->request.actual = req->request.length - req->remaining; in dwc3_gadget_ep_cleanup_completed_request()
3186 if (req->needs_extra_trb) { in dwc3_gadget_ep_cleanup_completed_request()
3189 req->needs_extra_trb = false; in dwc3_gadget_ep_cleanup_completed_request()
3204 list_for_each_entry_safe(req, tmp, &dep->started_list, list) { in dwc3_gadget_ep_cleanup_completed_requests()
3217 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_ep_should_continue()
3219 if (!dep->endpoint.desc || !dwc->pullups_connected || in dwc3_gadget_ep_should_continue()
3220 !dwc->connected) in dwc3_gadget_ep_should_continue()
3223 if (!list_empty(&dep->pending_list)) in dwc3_gadget_ep_should_continue()
3230 req = next_request(&dep->started_list); in dwc3_gadget_ep_should_continue()
3240 dep->frame_number = event->parameters; in dwc3_gadget_endpoint_frame_from_event()
3246 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_trbs_complete()
3251 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_trbs_complete()
3254 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && in dwc3_gadget_endpoint_trbs_complete()
3255 list_empty(&dep->started_list) && in dwc3_gadget_endpoint_trbs_complete()
3256 (list_empty(&dep->pending_list) || status == -EXDEV)) in dwc3_gadget_endpoint_trbs_complete()
3264 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. in dwc3_gadget_endpoint_trbs_complete()
3272 dep = dwc->eps[i]; in dwc3_gadget_endpoint_trbs_complete()
3274 if (!(dep->flags & DWC3_EP_ENABLED)) in dwc3_gadget_endpoint_trbs_complete()
3277 if (!list_empty(&dep->started_list)) in dwc3_gadget_endpoint_trbs_complete()
3281 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_endpoint_trbs_complete()
3282 reg |= dwc->u1u2; in dwc3_gadget_endpoint_trbs_complete()
3283 dwc3_writel(dwc->regs, DWC3_DCTL, reg); in dwc3_gadget_endpoint_trbs_complete()
3285 dwc->u1u2 = 0; in dwc3_gadget_endpoint_trbs_complete()
3296 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_transfer_in_progress()
3299 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_in_progress()
3300 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_in_progress()
3302 if (event->status & DEPEVT_STATUS_MISSED_ISOC) in dwc3_gadget_endpoint_transfer_in_progress()
3303 status = -EXDEV; in dwc3_gadget_endpoint_transfer_in_progress()
3313 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_transfer_complete()
3315 if (event->status & DEPEVT_STATUS_BUSERR) in dwc3_gadget_endpoint_transfer_complete()
3316 status = -ECONNRESET; in dwc3_gadget_endpoint_transfer_complete()
3319 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; in dwc3_gadget_endpoint_transfer_complete()
3335 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) in dwc3_gadget_endpoint_transfer_not_ready()
3344 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); in dwc3_gadget_endpoint_command_complete()
3349 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; in dwc3_gadget_endpoint_command_complete()
3350 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_gadget_endpoint_command_complete()
3353 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { in dwc3_gadget_endpoint_command_complete()
3354 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_command_complete()
3356 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; in dwc3_gadget_endpoint_command_complete()
3358 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; in dwc3_gadget_endpoint_command_complete()
3360 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); in dwc3_gadget_endpoint_command_complete()
3361 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3366 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); in dwc3_gadget_endpoint_command_complete()
3367 if (dwc->delayed_status) in dwc3_gadget_endpoint_command_complete()
3371 if ((dep->flags & DWC3_EP_DELAY_START) && in dwc3_gadget_endpoint_command_complete()
3372 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) in dwc3_gadget_endpoint_command_complete()
3375 dep->flags &= ~DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_command_complete()
3381 struct dwc3 *dwc = dep->dwc; in dwc3_gadget_endpoint_stream_event()
3383 if (event->status == DEPEVT_STREAMEVT_FOUND) { in dwc3_gadget_endpoint_stream_event()
3384 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3389 switch (event->parameters) { in dwc3_gadget_endpoint_stream_event()
3398 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { in dwc3_gadget_endpoint_stream_event()
3399 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) in dwc3_gadget_endpoint_stream_event()
3400 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; in dwc3_gadget_endpoint_stream_event()
3402 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; in dwc3_gadget_endpoint_stream_event()
3407 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || in dwc3_gadget_endpoint_stream_event()
3408 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || in dwc3_gadget_endpoint_stream_event()
3409 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)) in dwc3_gadget_endpoint_stream_event()
3429 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); in dwc3_gadget_endpoint_stream_event()
3431 dep->flags |= DWC3_EP_DELAY_START; in dwc3_gadget_endpoint_stream_event()
3439 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_gadget_endpoint_stream_event()
3446 u8 epnum = event->endpoint_number; in dwc3_endpoint_interrupt()
3448 dep = dwc->eps[epnum]; in dwc3_endpoint_interrupt()
3450 if (!(dep->flags & DWC3_EP_ENABLED)) { in dwc3_endpoint_interrupt()
3451 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) in dwc3_endpoint_interrupt()
3455 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) in dwc3_endpoint_interrupt()
3464 switch (event->endpoint_event) { in dwc3_endpoint_interrupt()
3487 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { in dwc3_disconnect_gadget()
3488 spin_unlock(&dwc->lock); in dwc3_disconnect_gadget()
3489 dwc->gadget_driver->disconnect(dwc->gadget); in dwc3_disconnect_gadget()
3490 spin_lock(&dwc->lock); in dwc3_disconnect_gadget()
3496 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { in dwc3_suspend_gadget()
3497 spin_unlock(&dwc->lock); in dwc3_suspend_gadget()
3498 dwc->gadget_driver->suspend(dwc->gadget); in dwc3_suspend_gadget()
3499 spin_lock(&dwc->lock); in dwc3_suspend_gadget()
3505 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_resume_gadget()
3506 spin_unlock(&dwc->lock); in dwc3_resume_gadget()
3507 dwc->gadget_driver->resume(dwc->gadget); in dwc3_resume_gadget()
3508 spin_lock(&dwc->lock); in dwc3_resume_gadget()
3514 if (!dwc->gadget_driver) in dwc3_reset_gadget()
3517 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { in dwc3_reset_gadget()
3518 spin_unlock(&dwc->lock); in dwc3_reset_gadget()
3519 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); in dwc3_reset_gadget()
3520 spin_lock(&dwc->lock); in dwc3_reset_gadget()
3531 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || in dwc3_stop_active_transfer()
3532 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) in dwc3_stop_active_transfer()
3565 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); in dwc3_stop_active_transfer()
3569 dep->resource_index = 0; in dwc3_stop_active_transfer()
3576 if (dep->stream_capable) in dwc3_stop_active_transfer()
3577 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; in dwc3_stop_active_transfer()
3580 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; in dwc3_stop_active_transfer()
3582 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; in dwc3_stop_active_transfer()
3593 dep = dwc->eps[epnum]; in dwc3_clear_stall_all_ep()
3597 if (!(dep->flags & DWC3_EP_STALL)) in dwc3_clear_stall_all_ep()
3600 dep->flags &= ~DWC3_EP_STALL; in dwc3_clear_stall_all_ep()
3613 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_disconnect_interrupt()
3620 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_disconnect_interrupt()
3621 dwc->setup_packet_pending = false; in dwc3_gadget_disconnect_interrupt()
3622 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); in dwc3_gadget_disconnect_interrupt()
3624 dwc->connected = false; in dwc3_gadget_disconnect_interrupt()
3633 * drivers to stop any active transfers through ep disable. in dwc3_gadget_reset_interrupt()
3634 * However, for functions which defer ep disable, such as mass in dwc3_gadget_reset_interrupt()
3638 dwc->connected = false; in dwc3_gadget_reset_interrupt()
3667 if (dwc->setup_packet_pending) in dwc3_gadget_reset_interrupt()
3674 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW in dwc3_gadget_reset_interrupt()
3679 dwc->connected = true; in dwc3_gadget_reset_interrupt()
3681 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_reset_interrupt()
3684 dwc->test_mode = false; in dwc3_gadget_reset_interrupt()
3688 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_reset_interrupt()
3690 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_reset_interrupt()
3701 reg = dwc3_readl(dwc->regs, DWC3_DSTS); in dwc3_gadget_conndone_interrupt()
3703 dwc->speed = speed; in dwc3_gadget_conndone_interrupt()
3708 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_conndone_interrupt()
3722 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
3723 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
3726 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; in dwc3_gadget_conndone_interrupt()
3728 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; in dwc3_gadget_conndone_interrupt()
3748 dwc->gadget->ep0->maxpacket = 512; in dwc3_gadget_conndone_interrupt()
3749 dwc->gadget->speed = USB_SPEED_SUPER; in dwc3_gadget_conndone_interrupt()
3752 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; in dwc3_gadget_conndone_interrupt()
3753 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; in dwc3_gadget_conndone_interrupt()
3758 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
3759 dwc->gadget->speed = USB_SPEED_HIGH; in dwc3_gadget_conndone_interrupt()
3763 dwc->gadget->ep0->maxpacket = 64; in dwc3_gadget_conndone_interrupt()
3764 dwc->gadget->speed = USB_SPEED_FULL; in dwc3_gadget_conndone_interrupt()
3768 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; in dwc3_gadget_conndone_interrupt()
3770 /* Enable USB2 LPM Capability */ in dwc3_gadget_conndone_interrupt()
3773 !dwc->usb2_gadget_lpm_disable && in dwc3_gadget_conndone_interrupt()
3776 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
3778 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
3780 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
3783 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | in dwc3_gadget_conndone_interrupt()
3784 (dwc->is_utmi_l1_suspend << 4)); in dwc3_gadget_conndone_interrupt()
3787 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and in dwc3_gadget_conndone_interrupt()
3789 * BESL value in the LPM token is less than or equal to LPM in dwc3_gadget_conndone_interrupt()
3792 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, in dwc3_gadget_conndone_interrupt()
3793 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); in dwc3_gadget_conndone_interrupt()
3795 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) in dwc3_gadget_conndone_interrupt()
3796 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); in dwc3_gadget_conndone_interrupt()
3800 if (dwc->usb2_gadget_lpm_disable) { in dwc3_gadget_conndone_interrupt()
3801 reg = dwc3_readl(dwc->regs, DWC3_DCFG); in dwc3_gadget_conndone_interrupt()
3803 dwc3_writel(dwc->regs, DWC3_DCFG, reg); in dwc3_gadget_conndone_interrupt()
3806 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_conndone_interrupt()
3811 dep = dwc->eps[0]; in dwc3_gadget_conndone_interrupt()
3814 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
3818 dep = dwc->eps[1]; in dwc3_gadget_conndone_interrupt()
3821 dev_err(dwc->dev, "failed to enable %s\n", dep->name); in dwc3_gadget_conndone_interrupt()
3841 if (dwc->async_callbacks && dwc->gadget_driver->resume) { in dwc3_gadget_wakeup_interrupt()
3842 spin_unlock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
3843 dwc->gadget_driver->resume(dwc->gadget); in dwc3_gadget_wakeup_interrupt()
3844 spin_lock(&dwc->lock); in dwc3_gadget_wakeup_interrupt()
3857 * host-initiated U3 exit. in dwc3_gadget_linksts_change_interrupt()
3868 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation in dwc3_gadget_linksts_change_interrupt()
3871 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); in dwc3_gadget_linksts_change_interrupt()
3874 if ((dwc->link_state == DWC3_LINK_STATE_U3) && in dwc3_gadget_linksts_change_interrupt()
3886 * suggested workaround is to disable DCTL[12:9] bits if we're in dwc3_gadget_linksts_change_interrupt()
3895 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us in dwc3_gadget_linksts_change_interrupt()
3903 switch (dwc->link_state) { in dwc3_gadget_linksts_change_interrupt()
3906 reg = dwc3_readl(dwc->regs, DWC3_DCTL); in dwc3_gadget_linksts_change_interrupt()
3912 if (!dwc->u1u2) in dwc3_gadget_linksts_change_interrupt()
3913 dwc->u1u2 = reg & u1u2; in dwc3_gadget_linksts_change_interrupt()
3928 if (dwc->speed == USB_SPEED_SUPER) in dwc3_gadget_linksts_change_interrupt()
3943 dwc->link_state = next; in dwc3_gadget_linksts_change_interrupt()
3951 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3) in dwc3_gadget_suspend_interrupt()
3954 dwc->link_state = next; in dwc3_gadget_suspend_interrupt()
3975 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER)) in dwc3_gadget_hibernation_interrupt()
3984 switch (event->type) { in dwc3_gadget_interrupt()
3998 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation, in dwc3_gadget_interrupt()
4002 dwc3_gadget_hibernation_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4005 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); in dwc3_gadget_interrupt()
4014 if (dwc->gadget->state >= USB_STATE_CONFIGURED) in dwc3_gadget_interrupt()
4016 event->event_info); in dwc3_gadget_interrupt()
4025 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); in dwc3_gadget_interrupt()
4032 trace_dwc3_event(event->raw, dwc); in dwc3_process_event_entry()
4034 if (!event->type.is_devspec) in dwc3_process_event_entry()
4035 dwc3_endpoint_interrupt(dwc, &event->depevt); in dwc3_process_event_entry()
4036 else if (event->type.type == DWC3_EVENT_TYPE_DEV) in dwc3_process_event_entry()
4037 dwc3_gadget_interrupt(dwc, &event->devt); in dwc3_process_event_entry()
4039 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); in dwc3_process_event_entry()
4044 struct dwc3 *dwc = evt->dwc; in dwc3_process_event_buf()
4049 left = evt->count; in dwc3_process_event_buf()
4051 if (!(evt->flags & DWC3_EVENT_PENDING)) in dwc3_process_event_buf()
4057 event.raw = *(u32 *) (evt->cache + evt->lpos); in dwc3_process_event_buf()
4070 evt->lpos = (evt->lpos + 4) % evt->length; in dwc3_process_event_buf()
4071 left -= 4; in dwc3_process_event_buf()
4074 evt->count = 0; in dwc3_process_event_buf()
4075 evt->flags &= ~DWC3_EVENT_PENDING; in dwc3_process_event_buf()
4079 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); in dwc3_process_event_buf()
4081 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); in dwc3_process_event_buf()
4083 if (dwc->imod_interval) { in dwc3_process_event_buf()
4084 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); in dwc3_process_event_buf()
4085 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); in dwc3_process_event_buf()
4094 struct dwc3 *dwc = evt->dwc; in dwc3_thread_interrupt()
4098 spin_lock_irqsave(&dwc->lock, flags); in dwc3_thread_interrupt()
4100 spin_unlock_irqrestore(&dwc->lock, flags); in dwc3_thread_interrupt()
4107 struct dwc3 *dwc = evt->dwc; in dwc3_check_event_buf()
4112 if (pm_runtime_suspended(dwc->dev)) { in dwc3_check_event_buf()
4113 pm_runtime_get(dwc->dev); in dwc3_check_event_buf()
4114 disable_irq_nosync(dwc->irq_gadget); in dwc3_check_event_buf()
4115 dwc->pending_events = true; in dwc3_check_event_buf()
4120 * With PCIe legacy interrupt, test shows that top-half irq handler can in dwc3_check_event_buf()
4121 * be called again after HW interrupt deassertion. Check if bottom-half in dwc3_check_event_buf()
4125 if (evt->flags & DWC3_EVENT_PENDING) in dwc3_check_event_buf()
4128 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); in dwc3_check_event_buf()
4133 evt->count = count; in dwc3_check_event_buf()
4134 evt->flags |= DWC3_EVENT_PENDING; in dwc3_check_event_buf()
4137 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); in dwc3_check_event_buf()
4139 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); in dwc3_check_event_buf()
4141 amount = min(count, evt->length - evt->lpos); in dwc3_check_event_buf()
4142 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); in dwc3_check_event_buf()
4145 memcpy(evt->cache, evt->buf, count - amount); in dwc3_check_event_buf()
4147 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); in dwc3_check_event_buf()
4161 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); in dwc3_gadget_get_irq()
4168 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4175 if (irq == -EPROBE_DEFER) in dwc3_gadget_get_irq()
4183 irq = -EINVAL; in dwc3_gadget_get_irq()
4197 * dwc3_gadget_init - initializes gadget related registers
4214 dwc->irq_gadget = irq; in dwc3_gadget_init()
4216 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, in dwc3_gadget_init()
4217 sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4218 &dwc->ep0_trb_addr, GFP_KERNEL); in dwc3_gadget_init()
4219 if (!dwc->ep0_trb) { in dwc3_gadget_init()
4220 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); in dwc3_gadget_init()
4221 ret = -ENOMEM; in dwc3_gadget_init()
4225 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); in dwc3_gadget_init()
4226 if (!dwc->setup_buf) { in dwc3_gadget_init()
4227 ret = -ENOMEM; in dwc3_gadget_init()
4231 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, in dwc3_gadget_init()
4232 &dwc->bounce_addr, GFP_KERNEL); in dwc3_gadget_init()
4233 if (!dwc->bounce) { in dwc3_gadget_init()
4234 ret = -ENOMEM; in dwc3_gadget_init()
4238 init_completion(&dwc->ep0_in_setup); in dwc3_gadget_init()
4239 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); in dwc3_gadget_init()
4240 if (!dwc->gadget) { in dwc3_gadget_init()
4241 ret = -ENOMEM; in dwc3_gadget_init()
4246 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); in dwc3_gadget_init()
4247 dev = &dwc->gadget->dev; in dwc3_gadget_init()
4248 dev->platform_data = dwc; in dwc3_gadget_init()
4249 dwc->gadget->ops = &dwc3_gadget_ops; in dwc3_gadget_init()
4250 dwc->gadget->speed = USB_SPEED_UNKNOWN; in dwc3_gadget_init()
4251 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; in dwc3_gadget_init()
4252 dwc->gadget->sg_supported = true; in dwc3_gadget_init()
4253 dwc->gadget->name = "dwc3-gadget"; in dwc3_gadget_init()
4254 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; in dwc3_gadget_init()
4270 * composite.c that we are USB 2.0 + LPM ECN. in dwc3_gadget_init()
4273 !dwc->dis_metastability_quirk) in dwc3_gadget_init()
4274 dev_info(dwc->dev, "changing max_speed on rev %08x\n", in dwc3_gadget_init()
4275 dwc->revision); in dwc3_gadget_init()
4277 dwc->gadget->max_speed = dwc->maximum_speed; in dwc3_gadget_init()
4278 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; in dwc3_gadget_init()
4285 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); in dwc3_gadget_init()
4289 ret = usb_add_gadget(dwc->gadget); in dwc3_gadget_init()
4291 dev_err(dwc->dev, "failed to add gadget\n"); in dwc3_gadget_init()
4295 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) in dwc3_gadget_init()
4296 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); in dwc3_gadget_init()
4298 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); in dwc3_gadget_init()
4305 usb_put_gadget(dwc->gadget); in dwc3_gadget_init()
4306 dwc->gadget = NULL; in dwc3_gadget_init()
4308 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_init()
4309 dwc->bounce_addr); in dwc3_gadget_init()
4312 kfree(dwc->setup_buf); in dwc3_gadget_init()
4315 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_init()
4316 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_init()
4322 /* -------------------------------------------------------------------------- */
4326 if (!dwc->gadget) in dwc3_gadget_exit()
4329 usb_del_gadget(dwc->gadget); in dwc3_gadget_exit()
4331 usb_put_gadget(dwc->gadget); in dwc3_gadget_exit()
4332 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, in dwc3_gadget_exit()
4333 dwc->bounce_addr); in dwc3_gadget_exit()
4334 kfree(dwc->setup_buf); in dwc3_gadget_exit()
4335 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, in dwc3_gadget_exit()
4336 dwc->ep0_trb, dwc->ep0_trb_addr); in dwc3_gadget_exit()
4341 if (!dwc->gadget_driver) in dwc3_gadget_suspend()
4355 if (!dwc->gadget_driver) in dwc3_gadget_resume()
4377 if (dwc->pending_events) { in dwc3_gadget_process_pending_events()
4378 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf); in dwc3_gadget_process_pending_events()
4379 dwc->pending_events = false; in dwc3_gadget_process_pending_events()
4380 enable_irq(dwc->irq_gadget); in dwc3_gadget_process_pending_events()