30 static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep);
31 static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,
38 	struct dwc3			*dwc;  in dwc3_ep0_prepare_one_trb()  local
40 	dwc = dep->dwc;  in dwc3_ep0_prepare_one_trb()
41 	trb = &dwc->ep0_trb[dep->trb_enqueue];  in dwc3_ep0_prepare_one_trb()
66 	struct dwc3			*dwc;  in dwc3_ep0_start_trans()  local
72 	dwc = dep->dwc;  in dwc3_ep0_start_trans()
75 	params.param0 = upper_32_bits(dwc->ep0_trb_addr);  in dwc3_ep0_start_trans()
76 	params.param1 = lower_32_bits(dwc->ep0_trb_addr);  in dwc3_ep0_start_trans()
82 	dwc->ep0_next_event = DWC3_EP0_COMPLETE;  in dwc3_ep0_start_trans()
90 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep0_queue()  local
112 		if (dwc->ep0state != EP0_DATA_PHASE) {  in __dwc3_gadget_ep0_queue()
113 			dev_WARN(dwc->dev, "Unexpected pending request\n");  in __dwc3_gadget_ep0_queue()
117 		__dwc3_ep0_do_control_data(dwc, dwc->eps[direction], req);  in __dwc3_gadget_ep0_queue()
129 	if (dwc->delayed_status) {  in __dwc3_gadget_ep0_queue()
132 		direction = !dwc->ep0_expect_in;  in __dwc3_gadget_ep0_queue()
133 		dwc->delayed_status = false;  in __dwc3_gadget_ep0_queue()
134 		usb_gadget_set_state(&dwc->gadget, USB_STATE_CONFIGURED);  in __dwc3_gadget_ep0_queue()
136 		if (dwc->ep0state == EP0_STATUS_PHASE)  in __dwc3_gadget_ep0_queue()
137 			__dwc3_ep0_do_control_status(dwc, dwc->eps[direction]);  in __dwc3_gadget_ep0_queue()
174 	if (dwc->three_stage_setup) {  in __dwc3_gadget_ep0_queue()
177 		direction = dwc->ep0_expect_in;  in __dwc3_gadget_ep0_queue()
178 		dwc->ep0state = EP0_DATA_PHASE;  in __dwc3_gadget_ep0_queue()
180 		__dwc3_ep0_do_control_data(dwc, dwc->eps[direction], req);  in __dwc3_gadget_ep0_queue()
193 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep0_queue()  local
199 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep0_queue()
201 		dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",  in dwc3_gadget_ep0_queue()
216 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep0_queue()
221 static void dwc3_ep0_stall_and_restart(struct dwc3 *dwc)  in dwc3_ep0_stall_and_restart()  argument
226 	dep = dwc->eps[1];  in dwc3_ep0_stall_and_restart()
230 	dep = dwc->eps[0];  in dwc3_ep0_stall_and_restart()
233 	dwc->delayed_status = false;  in dwc3_ep0_stall_and_restart()
242 	dwc->ep0state = EP0_SETUP_PHASE;  in dwc3_ep0_stall_and_restart()
243 	dwc3_ep0_out_start(dwc);  in dwc3_ep0_stall_and_restart()
249 	struct dwc3			*dwc = dep->dwc;  in __dwc3_gadget_ep0_set_halt()  local
251 	dwc3_ep0_stall_and_restart(dwc);  in __dwc3_gadget_ep0_set_halt()
259 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep0_set_halt()  local
263 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep0_set_halt()
265 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep0_set_halt()
270 void dwc3_ep0_out_start(struct dwc3 *dwc)  in dwc3_ep0_out_start()  argument
275 	complete(&dwc->ep0_in_setup);  in dwc3_ep0_out_start()
277 	dep = dwc->eps[0];  in dwc3_ep0_out_start()
278 	dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 8,  in dwc3_ep0_out_start()
284 static struct dwc3_ep *dwc3_wIndex_to_dep(struct dwc3 *dwc, __le16 wIndex_le)  in dwc3_wIndex_to_dep()  argument
294 	dep = dwc->eps[epnum];  in dwc3_wIndex_to_dep()
307 static int dwc3_ep0_handle_status(struct dwc3 *dwc,  in dwc3_ep0_handle_status()  argument
328 		usb_status |= dwc->gadget.is_selfpowered;  in dwc3_ep0_handle_status()
330 		if ((dwc->speed == DWC3_DSTS_SUPERSPEED) ||  in dwc3_ep0_handle_status()
331 		    (dwc->speed == DWC3_DSTS_SUPERSPEED_PLUS)) {  in dwc3_ep0_handle_status()
332 			reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_ep0_handle_status()
349 		dep = dwc3_wIndex_to_dep(dwc, ctrl->wIndex);  in dwc3_ep0_handle_status()
360 	response_pkt = (__le16 *) dwc->setup_buf;  in dwc3_ep0_handle_status()
363 	dep = dwc->eps[0];  in dwc3_ep0_handle_status()
364 	dwc->ep0_usb_req.dep = dep;  in dwc3_ep0_handle_status()
365 	dwc->ep0_usb_req.request.length = sizeof(*response_pkt);  in dwc3_ep0_handle_status()
366 	dwc->ep0_usb_req.request.buf = dwc->setup_buf;  in dwc3_ep0_handle_status()
367 	dwc->ep0_usb_req.request.complete = dwc3_ep0_status_cmpl;  in dwc3_ep0_handle_status()
369 	return __dwc3_gadget_ep0_queue(dep, &dwc->ep0_usb_req);  in dwc3_ep0_handle_status()
372 static int dwc3_ep0_handle_u1(struct dwc3 *dwc, enum usb_device_state state,  in dwc3_ep0_handle_u1()  argument
379 	if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&  in dwc3_ep0_handle_u1()
380 			(dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))  in dwc3_ep0_handle_u1()
382 	if (set && dwc->dis_u1_entry_quirk)  in dwc3_ep0_handle_u1()
385 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_ep0_handle_u1()
390 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_ep0_handle_u1()
395 static int dwc3_ep0_handle_u2(struct dwc3 *dwc, enum usb_device_state state,  in dwc3_ep0_handle_u2()  argument
403 	if ((dwc->speed != DWC3_DSTS_SUPERSPEED) &&  in dwc3_ep0_handle_u2()
404 			(dwc->speed != DWC3_DSTS_SUPERSPEED_PLUS))  in dwc3_ep0_handle_u2()
406 	if (set && dwc->dis_u2_entry_quirk)  in dwc3_ep0_handle_u2()
409 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_ep0_handle_u2()
414 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_ep0_handle_u2()
419 static int dwc3_ep0_handle_test(struct dwc3 *dwc, enum usb_device_state state,  in dwc3_ep0_handle_test()  argument
433 		dwc->test_mode_nr = wIndex >> 8;  in dwc3_ep0_handle_test()
434 		dwc->test_mode = true;  in dwc3_ep0_handle_test()
443 static int dwc3_ep0_handle_device(struct dwc3 *dwc,  in dwc3_ep0_handle_device()  argument
453 	state = dwc->gadget.state;  in dwc3_ep0_handle_device()
463 		ret = dwc3_ep0_handle_u1(dwc, state, set);  in dwc3_ep0_handle_device()
466 		ret = dwc3_ep0_handle_u2(dwc, state, set);  in dwc3_ep0_handle_device()
472 		ret = dwc3_ep0_handle_test(dwc, state, wIndex, set);  in dwc3_ep0_handle_device()
481 static int dwc3_ep0_handle_intf(struct dwc3 *dwc,  in dwc3_ep0_handle_intf()  argument
506 static int dwc3_ep0_handle_endpoint(struct dwc3 *dwc,  in dwc3_ep0_handle_endpoint()  argument
517 		dep = dwc3_wIndex_to_dep(dwc, ctrl->wIndex);  in dwc3_ep0_handle_endpoint()
535 static int dwc3_ep0_handle_feature(struct dwc3 *dwc,  in dwc3_ep0_handle_feature()  argument
545 		ret = dwc3_ep0_handle_device(dwc, ctrl, set);  in dwc3_ep0_handle_feature()
548 		ret = dwc3_ep0_handle_intf(dwc, ctrl, set);  in dwc3_ep0_handle_feature()
551 		ret = dwc3_ep0_handle_endpoint(dwc, ctrl, set);  in dwc3_ep0_handle_feature()
560 static int dwc3_ep0_set_address(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_set_address()  argument
562 	enum usb_device_state state = dwc->gadget.state;  in dwc3_ep0_set_address()
568 		dev_err(dwc->dev, "invalid device address %d\n", addr);  in dwc3_ep0_set_address()
573 		dev_err(dwc->dev, "can't SetAddress() from Configured State\n");  in dwc3_ep0_set_address()
577 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in dwc3_ep0_set_address()
580 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in dwc3_ep0_set_address()
583 		usb_gadget_set_state(&dwc->gadget, USB_STATE_ADDRESS);  in dwc3_ep0_set_address()
585 		usb_gadget_set_state(&dwc->gadget, USB_STATE_DEFAULT);  in dwc3_ep0_set_address()
590 static int dwc3_ep0_delegate_req(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_delegate_req()  argument
594 	spin_unlock(&dwc->lock);  in dwc3_ep0_delegate_req()
595 	ret = dwc->gadget_driver->setup(&dwc->gadget, ctrl);  in dwc3_ep0_delegate_req()
596 	spin_lock(&dwc->lock);  in dwc3_ep0_delegate_req()
600 static int dwc3_ep0_set_config(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_set_config()  argument
602 	enum usb_device_state state = dwc->gadget.state;  in dwc3_ep0_set_config()
614 		ret = dwc3_ep0_delegate_req(dwc, ctrl);  in dwc3_ep0_set_config()
625 				usb_gadget_set_state(&dwc->gadget,  in dwc3_ep0_set_config()
632 			reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_ep0_set_config()
633 			if (!dwc->dis_u1_entry_quirk)  in dwc3_ep0_set_config()
635 			if (!dwc->dis_u2_entry_quirk)  in dwc3_ep0_set_config()
637 			dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_ep0_set_config()
642 		ret = dwc3_ep0_delegate_req(dwc, ctrl);  in dwc3_ep0_set_config()
644 			usb_gadget_set_state(&dwc->gadget,  in dwc3_ep0_set_config()
656 	struct dwc3	*dwc = dep->dwc;  in dwc3_ep0_set_sel_cmpl()  local
672 	dwc->u1sel = timing.u1sel;  in dwc3_ep0_set_sel_cmpl()
673 	dwc->u1pel = timing.u1pel;  in dwc3_ep0_set_sel_cmpl()
674 	dwc->u2sel = le16_to_cpu(timing.u2sel);  in dwc3_ep0_set_sel_cmpl()
675 	dwc->u2pel = le16_to_cpu(timing.u2pel);  in dwc3_ep0_set_sel_cmpl()
677 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_ep0_set_sel_cmpl()
679 		param = dwc->u2pel;  in dwc3_ep0_set_sel_cmpl()
681 		param = dwc->u1pel;  in dwc3_ep0_set_sel_cmpl()
692 	ret = dwc3_send_gadget_generic_command(dwc,  in dwc3_ep0_set_sel_cmpl()
697 static int dwc3_ep0_set_sel(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_set_sel()  argument
700 	enum usb_device_state state = dwc->gadget.state;  in dwc3_ep0_set_sel()
709 		dev_err(dwc->dev, "Set SEL should be 6 bytes, got %d\n",  in dwc3_ep0_set_sel()
722 	dep = dwc->eps[0];  in dwc3_ep0_set_sel()
723 	dwc->ep0_usb_req.dep = dep;  in dwc3_ep0_set_sel()
724 	dwc->ep0_usb_req.request.length = dep->endpoint.maxpacket;  in dwc3_ep0_set_sel()
725 	dwc->ep0_usb_req.request.buf = dwc->setup_buf;  in dwc3_ep0_set_sel()
726 	dwc->ep0_usb_req.request.complete = dwc3_ep0_set_sel_cmpl;  in dwc3_ep0_set_sel()
728 	return __dwc3_gadget_ep0_queue(dep, &dwc->ep0_usb_req);  in dwc3_ep0_set_sel()
731 static int dwc3_ep0_set_isoch_delay(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_set_isoch_delay()  argument
744 	dwc->gadget.isoch_delay = wValue;  in dwc3_ep0_set_isoch_delay()
749 static int dwc3_ep0_std_request(struct dwc3 *dwc, struct usb_ctrlrequest *ctrl)  in dwc3_ep0_std_request()  argument
755 		ret = dwc3_ep0_handle_status(dwc, ctrl);  in dwc3_ep0_std_request()
758 		ret = dwc3_ep0_handle_feature(dwc, ctrl, 0);  in dwc3_ep0_std_request()
761 		ret = dwc3_ep0_handle_feature(dwc, ctrl, 1);  in dwc3_ep0_std_request()
764 		ret = dwc3_ep0_set_address(dwc, ctrl);  in dwc3_ep0_std_request()
767 		ret = dwc3_ep0_set_config(dwc, ctrl);  in dwc3_ep0_std_request()
770 		ret = dwc3_ep0_set_sel(dwc, ctrl);  in dwc3_ep0_std_request()
773 		ret = dwc3_ep0_set_isoch_delay(dwc, ctrl);  in dwc3_ep0_std_request()
776 		ret = dwc3_ep0_delegate_req(dwc, ctrl);  in dwc3_ep0_std_request()
783 static void dwc3_ep0_inspect_setup(struct dwc3 *dwc,  in dwc3_ep0_inspect_setup()  argument
786 	struct usb_ctrlrequest *ctrl = (void *) dwc->ep0_trb;  in dwc3_ep0_inspect_setup()
790 	if (!dwc->gadget_driver)  in dwc3_ep0_inspect_setup()
797 		dwc->three_stage_setup = false;  in dwc3_ep0_inspect_setup()
798 		dwc->ep0_expect_in = false;  in dwc3_ep0_inspect_setup()
799 		dwc->ep0_next_event = DWC3_EP0_NRDY_STATUS;  in dwc3_ep0_inspect_setup()
801 		dwc->three_stage_setup = true;  in dwc3_ep0_inspect_setup()
802 		dwc->ep0_expect_in = !!(ctrl->bRequestType & USB_DIR_IN);  in dwc3_ep0_inspect_setup()
803 		dwc->ep0_next_event = DWC3_EP0_NRDY_DATA;  in dwc3_ep0_inspect_setup()
807 		ret = dwc3_ep0_std_request(dwc, ctrl);  in dwc3_ep0_inspect_setup()
809 		ret = dwc3_ep0_delegate_req(dwc, ctrl);  in dwc3_ep0_inspect_setup()
812 		dwc->delayed_status = true;  in dwc3_ep0_inspect_setup()
816 		dwc3_ep0_stall_and_restart(dwc);  in dwc3_ep0_inspect_setup()
819 static void dwc3_ep0_complete_data(struct dwc3 *dwc,  in dwc3_ep0_complete_data()  argument
832 	ep0 = dwc->eps[0];  in dwc3_ep0_complete_data()
834 	dwc->ep0_next_event = DWC3_EP0_NRDY_STATUS;  in dwc3_ep0_complete_data()
835 	trb = dwc->ep0_trb;  in dwc3_ep0_complete_data()
844 		dwc->setup_packet_pending = true;  in dwc3_ep0_complete_data()
858 	     ur->length && ur->zero) || dwc->ep0_bounced) {  in dwc3_ep0_complete_data()
864 			dwc->eps[1]->trb_enqueue = 0;  in dwc3_ep0_complete_data()
866 			dwc->eps[0]->trb_enqueue = 0;  in dwc3_ep0_complete_data()
868 		dwc->ep0_bounced = false;  in dwc3_ep0_complete_data()
872 		dwc3_ep0_stall_and_restart(dwc);  in dwc3_ep0_complete_data()
877 static void dwc3_ep0_complete_status(struct dwc3 *dwc,  in dwc3_ep0_complete_status()  argument
885 	dep = dwc->eps[0];  in dwc3_ep0_complete_status()
886 	trb = dwc->ep0_trb;  in dwc3_ep0_complete_status()
896 	if (dwc->test_mode) {  in dwc3_ep0_complete_status()
899 		ret = dwc3_gadget_set_test_mode(dwc, dwc->test_mode_nr);  in dwc3_ep0_complete_status()
901 			dev_err(dwc->dev, "invalid test #%d\n",  in dwc3_ep0_complete_status()
902 					dwc->test_mode_nr);  in dwc3_ep0_complete_status()
903 			dwc3_ep0_stall_and_restart(dwc);  in dwc3_ep0_complete_status()
910 		dwc->setup_packet_pending = true;  in dwc3_ep0_complete_status()
912 	dwc->ep0state = EP0_SETUP_PHASE;  in dwc3_ep0_complete_status()
913 	dwc3_ep0_out_start(dwc);  in dwc3_ep0_complete_status()
916 static void dwc3_ep0_xfer_complete(struct dwc3 *dwc,  in dwc3_ep0_xfer_complete()  argument
919 	struct dwc3_ep		*dep = dwc->eps[event->endpoint_number];  in dwc3_ep0_xfer_complete()
923 	dwc->setup_packet_pending = false;  in dwc3_ep0_xfer_complete()
925 	switch (dwc->ep0state) {  in dwc3_ep0_xfer_complete()
927 		dwc3_ep0_inspect_setup(dwc, event);  in dwc3_ep0_xfer_complete()
931 		dwc3_ep0_complete_data(dwc, event);  in dwc3_ep0_xfer_complete()
935 		dwc3_ep0_complete_status(dwc, event);  in dwc3_ep0_xfer_complete()
938 		WARN(true, "UNKNOWN ep0state %d\n", dwc->ep0state);  in dwc3_ep0_xfer_complete()
942 static void __dwc3_ep0_do_control_data(struct dwc3 *dwc,  in __dwc3_ep0_do_control_data()  argument
950 		dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 0,  in __dwc3_ep0_do_control_data()
958 		ret = usb_gadget_map_request_by_dev(dwc->sysdev,  in __dwc3_ep0_do_control_data()
965 		dwc->ep0_bounced = true;  in __dwc3_ep0_do_control_data()
973 		req->trb = &dwc->ep0_trb[dep->trb_enqueue - 1];  in __dwc3_ep0_do_control_data()
976 		dwc3_ep0_prepare_one_trb(dep, dwc->bounce_addr,  in __dwc3_ep0_do_control_data()
984 		ret = usb_gadget_map_request_by_dev(dwc->sysdev,  in __dwc3_ep0_do_control_data()
995 		req->trb = &dwc->ep0_trb[dep->trb_enqueue - 1];  in __dwc3_ep0_do_control_data()
998 		dwc3_ep0_prepare_one_trb(dep, dwc->bounce_addr,  in __dwc3_ep0_do_control_data()
1003 		ret = usb_gadget_map_request_by_dev(dwc->sysdev,  in __dwc3_ep0_do_control_data()
1012 		req->trb = &dwc->ep0_trb[dep->trb_enqueue];  in __dwc3_ep0_do_control_data()
1022 	struct dwc3		*dwc = dep->dwc;  in dwc3_ep0_start_control_status()  local
1025 	type = dwc->three_stage_setup ? DWC3_TRBCTL_CONTROL_STATUS3  in dwc3_ep0_start_control_status()
1028 	dwc3_ep0_prepare_one_trb(dep, dwc->ep0_trb_addr, 0, type, false);  in dwc3_ep0_start_control_status()
1032 static void __dwc3_ep0_do_control_status(struct dwc3 *dwc, struct dwc3_ep *dep)  in __dwc3_ep0_do_control_status()  argument
1037 static void dwc3_ep0_do_control_status(struct dwc3 *dwc,  in dwc3_ep0_do_control_status()  argument
1040 	struct dwc3_ep		*dep = dwc->eps[event->endpoint_number];  in dwc3_ep0_do_control_status()
1042 	__dwc3_ep0_do_control_status(dwc, dep);  in dwc3_ep0_do_control_status()
1045 static void dwc3_ep0_end_control_data(struct dwc3 *dwc, struct dwc3_ep *dep)  in dwc3_ep0_end_control_data()  argument
1063 static void dwc3_ep0_xfernotready(struct dwc3 *dwc,  in dwc3_ep0_xfernotready()  argument
1077 		if (dwc->ep0_expect_in != event->endpoint_number) {  in dwc3_ep0_xfernotready()
1078 			struct dwc3_ep	*dep = dwc->eps[dwc->ep0_expect_in];  in dwc3_ep0_xfernotready()
1080 			dev_err(dwc->dev, "unexpected direction for Data Phase\n");  in dwc3_ep0_xfernotready()
1081 			dwc3_ep0_end_control_data(dwc, dep);  in dwc3_ep0_xfernotready()
1082 			dwc3_ep0_stall_and_restart(dwc);  in dwc3_ep0_xfernotready()
1089 		if (dwc->ep0_next_event != DWC3_EP0_NRDY_STATUS)  in dwc3_ep0_xfernotready()
1092 		dwc->ep0state = EP0_STATUS_PHASE;  in dwc3_ep0_xfernotready()
1094 		if (dwc->delayed_status) {  in dwc3_ep0_xfernotready()
1095 			struct dwc3_ep *dep = dwc->eps[0];  in dwc3_ep0_xfernotready()
1104 				dwc->delayed_status = false;  in dwc3_ep0_xfernotready()
1105 				usb_gadget_set_state(&dwc->gadget,  in dwc3_ep0_xfernotready()
1107 				dwc3_ep0_do_control_status(dwc, event);  in dwc3_ep0_xfernotready()
1113 		dwc3_ep0_do_control_status(dwc, event);  in dwc3_ep0_xfernotready()
1117 void dwc3_ep0_interrupt(struct dwc3 *dwc,  in dwc3_ep0_interrupt()  argument
1122 		dwc3_ep0_xfer_complete(dwc, event);  in dwc3_ep0_interrupt()
1126 		dwc3_ep0_xfernotready(dwc, event);  in dwc3_ep0_interrupt()