1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
3  * hcd_queue.c - DesignWare HS OTG Controller host queuing routines
5  * Copyright (C) 2004-2013 Synopsys, Inc.
16  * 3. The names of the above-listed copyright holders may not be used
47 #include <linux/dma-mapping.h>
65  * dwc2_periodic_channel_available() - Checks that a channel is available for a
77 	 * non-periodic transactions  in dwc2_periodic_channel_available()
82 	num_channels = hsotg->params.host_channels;  in dwc2_periodic_channel_available()
83 	if ((hsotg->periodic_channels + hsotg->non_periodic_channels <  in dwc2_periodic_channel_available()
84 	     num_channels) && (hsotg->periodic_channels < num_channels - 1)) {  in dwc2_periodic_channel_available()
87 		dev_dbg(hsotg->dev,  in dwc2_periodic_channel_available()
88 			"%s: Total channels: %d, Periodic: %d, Non-periodic: %d\n",  in dwc2_periodic_channel_available()
90 			hsotg->periodic_channels, hsotg->non_periodic_channels);  in dwc2_periodic_channel_available()
91 		status = -ENOSPC;  in dwc2_periodic_channel_available()
98  * dwc2_check_periodic_bandwidth() - Checks that there is sufficient bandwidth
107  * periodic schedule may occur in the same (micro)frame
117 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {  in dwc2_check_periodic_bandwidth()
122 		max_claimed_usecs = 100 - qh->host_us;  in dwc2_check_periodic_bandwidth()
125 		 * Full speed mode  in dwc2_check_periodic_bandwidth()
128 		max_claimed_usecs = 900 - qh->host_us;  in dwc2_check_periodic_bandwidth()
131 	if (hsotg->periodic_usecs > max_claimed_usecs) {  in dwc2_check_periodic_bandwidth()
132 		dev_err(hsotg->dev,  in dwc2_check_periodic_bandwidth()
134 			__func__, hsotg->periodic_usecs, qh->host_us);  in dwc2_check_periodic_bandwidth()
135 		status = -ENOSPC;  in dwc2_check_periodic_bandwidth()
142  * pmap_schedule() - Schedule time in a periodic bitmap (pmap).
164  * want the same resource.  The resource is divided into fixed-sized periods
170  * but does mean that we need to handle things specially (and non-ideally) if
175  * - If interval is 1, we need to take up space in each of the 8
177  * - If interval is 2, we need to take up space in half of the
179  * - If interval is 3, we actually need to fall back to interval 1.
183  *   0, 3, and 6.  Since we could be in any frame we need to reserve
187  * - If interval is 4, easy.
188  * - If interval is 5, we again need interval 1.  The schedule will be
190  * - If interval is 6, we need interval 2.  0, 6, 4, 2.
191  * - If interval is 7, we need interval 1.
192  * - If interval is 8, we need interval 8.
197  * Note that at the moment this function tends to front-pack the schedule.
198  * In some cases that's really non-ideal (it's hard to schedule things that
229  * Returns either -ENOSPC or a >= 0 start bit which should be passed to the
230  * unschedule routine.  The map bitmap will be updated on a non-error result.
242 		return -ENOSPC;  in pmap_schedule()
252 		return -ENOSPC;  in pmap_schedule()
309 				start = ith_start - interval_bits * i;  in pmap_schedule()
319 		return -ENOSPC;  in pmap_schedule()
331  * pmap_unschedule() - Undo work done by pmap_schedule()
362  * dwc2_get_ls_map() - Get the map used for the given qh
368  * running the host straight in low speed / full speed mode it appears as if
370  * add logic here to get a map out of "hsotg" if !qh->do_split.
380 	if (WARN_ON(!qh->dwc_tt))  in dwc2_get_ls_map()
384 	map = qh->dwc_tt->periodic_bitmaps;  in dwc2_get_ls_map()
385 	if (qh->dwc_tt->usb_tt->multi)  in dwc2_get_ls_map()
386 		map += DWC2_ELEMENTS_PER_LS_BITMAP * (qh->ttport - 1);  in dwc2_get_ls_map()
393  * cat_printf() - A printf() + strcat() helper
418 		(*buf)[*size - 1] = '\0';  in cat_printf()
423 		*size -= i;  in cat_printf()
428  * pmap_print() - Print the given periodic map
465 					start = i - period_start;  in pmap_print()
481 			cat_printf(&buf, &buf_size, "%d %s -%3d %s", start,  in pmap_print()
482 				   units, start + count - 1, units);  in pmap_print()
497  * dwc2_qh_print() - Helper function for dwc2_qh_schedule_print()
506 	dwc2_sch_dbg(print_data->hsotg, "QH=%p ...%s\n", print_data->qh, str);  in dwc2_qh_print()
510  * dwc2_qh_schedule_print() - Print the periodic schedule
527 	if (qh->schedule_low_speed) {  in dwc2_qh_schedule_print()
531 			     qh, qh->device_us,  in dwc2_qh_schedule_print()
532 			     DWC2_ROUND_US_TO_SLICE(qh->device_us),  in dwc2_qh_schedule_print()
533 			     DWC2_US_PER_SLICE * qh->ls_start_schedule_slice);  in dwc2_qh_schedule_print()
537 				     "QH=%p Whole low/full speed map %p now:\n",  in dwc2_qh_schedule_print()
540 				   DWC2_LS_SCHEDULE_FRAMES, "Frame ", "slices",  in dwc2_qh_schedule_print()
545 	for (i = 0; i < qh->num_hs_transfers; i++) {  in dwc2_qh_schedule_print()
546 		struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + i;  in dwc2_qh_schedule_print()
547 		int uframe = trans_time->start_schedule_us /  in dwc2_qh_schedule_print()
549 		int rel_us = trans_time->start_schedule_us %  in dwc2_qh_schedule_print()
554 			     qh, i, trans_time->duration_us, uframe, rel_us);  in dwc2_qh_schedule_print()
556 	if (qh->num_hs_transfers) {  in dwc2_qh_schedule_print()
558 		pmap_print(hsotg->hs_periodic_bitmap,  in dwc2_qh_schedule_print()
570  * dwc2_ls_pmap_schedule() - Schedule a low speed QH
587 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);  in dwc2_ls_pmap_schedule()
592 		return -EINVAL;  in dwc2_ls_pmap_schedule()
608 			      qh->device_interval, search_slice, false);  in dwc2_ls_pmap_schedule()
613 	qh->ls_start_schedule_slice = slice;  in dwc2_ls_pmap_schedule()
618  * dwc2_ls_pmap_unschedule() - Undo work done by dwc2_ls_pmap_schedule()
626 	int slices = DIV_ROUND_UP(qh->device_us, DWC2_US_PER_SLICE);  in dwc2_ls_pmap_unschedule()
634 			DWC2_LS_SCHEDULE_FRAMES, slices, qh->device_interval,  in dwc2_ls_pmap_unschedule()
635 			qh->ls_start_schedule_slice);  in dwc2_ls_pmap_unschedule()
639  * dwc2_hs_pmap_schedule - Schedule in the main high speed schedule
643  * We'll start looking in qh->hs_transfers[index].start_schedule_us.  We'll
653  * @index:           The index into qh->hs_transfers that we're working with.
661 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;  in dwc2_hs_pmap_schedule()
664 	us = pmap_schedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_schedule()
666 			   DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,  in dwc2_hs_pmap_schedule()
667 			   qh->host_interval, trans_time->start_schedule_us,  in dwc2_hs_pmap_schedule()
673 	trans_time->start_schedule_us = us;  in dwc2_hs_pmap_schedule()
678  * dwc2_ls_pmap_unschedule() - Undo work done by dwc2_hs_pmap_schedule()
687 	struct dwc2_hs_transfer_time *trans_time = qh->hs_transfers + index;  in dwc2_hs_pmap_unschedule()
689 	pmap_unschedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_unschedule()
691 			DWC2_HS_SCHEDULE_UFRAMES, trans_time->duration_us,  in dwc2_hs_pmap_unschedule()
692 			qh->host_interval, trans_time->start_schedule_us);  in dwc2_hs_pmap_unschedule()
696  * dwc2_uframe_schedule_split - Schedule a QH for a periodic split xfer.
703  * device is in low or full speed mode (and behind a hub).
711 	int bytecount = qh->maxp_mult * qh->maxp;  in dwc2_uframe_schedule_split()
720 	host_interval_in_sched = gcd(qh->host_interval,  in dwc2_uframe_schedule_split()
730 	 * Note that this will tend to front-load the high speed schedule.  in dwc2_uframe_schedule_split()
748 		if (qh->schedule_low_speed) {  in dwc2_uframe_schedule_split()
765 		 * This will give us a number 0 - 7 if  in dwc2_uframe_schedule_split()
766 		 * DWC2_LS_SCHEDULE_FRAMES == 1, or 0 - 15 if == 2, or ...  in dwc2_uframe_schedule_split()
768 		start_s_uframe = qh->ls_start_schedule_slice /  in dwc2_uframe_schedule_split()
771 		/* Get a number that's always 0 - 7 */  in dwc2_uframe_schedule_split()
777 		 * Move on to the next full frame (assuming there is one).  in dwc2_uframe_schedule_split()
783 			if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
786 				(qh->ls_start_schedule_slice /  in dwc2_uframe_schedule_split()
794 		 * - start split            (frame -1)  in dwc2_uframe_schedule_split()
795 		 * - complete split w/ data (frame +1)  in dwc2_uframe_schedule_split()
796 		 * - complete split w/ data (frame +2)  in dwc2_uframe_schedule_split()
797 		 * - ...  in dwc2_uframe_schedule_split()
798 		 * - complete split w/ data (frame +num_data_packets)  in dwc2_uframe_schedule_split()
799 		 * - complete split w/ data (frame +num_data_packets+1)  in dwc2_uframe_schedule_split()
800 		 * - complete split w/ data (frame +num_data_packets+2, max 8)  in dwc2_uframe_schedule_split()
801 		 *   ...though if frame was "0" then max is 7...  in dwc2_uframe_schedule_split()
804 		 * - start split w/ data    (frame -1)  in dwc2_uframe_schedule_split()
805 		 * - start split w/ data    (frame +0)  in dwc2_uframe_schedule_split()
806 		 * - ...  in dwc2_uframe_schedule_split()
807 		 * - start split w/ data    (frame +num_data_packets-2)  in dwc2_uframe_schedule_split()
810 		 * - start split            (frame -1)  in dwc2_uframe_schedule_split()
811 		 * - complete split w/ data (frame +1)  in dwc2_uframe_schedule_split()
812 		 * - complete split w/ data (frame +2)  in dwc2_uframe_schedule_split()
813 		 * - complete split w/ data (frame +3, max 8)  in dwc2_uframe_schedule_split()
816 		 * - start split w/ data    (frame -1)  in dwc2_uframe_schedule_split()
817 		 * - complete split         (frame +1)  in dwc2_uframe_schedule_split()
818 		 * - complete split         (frame +2)  in dwc2_uframe_schedule_split()
819 		 * - complete split         (frame +3, max 8)  in dwc2_uframe_schedule_split()
824 				   host_interval_in_sched - 1) %  in dwc2_uframe_schedule_split()
826 		if (qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in)  in dwc2_uframe_schedule_split()
832 		first_data_bytes = 188 -  in dwc2_uframe_schedule_split()
833 			DIV_ROUND_UP(188 * (qh->ls_start_schedule_slice %  in dwc2_uframe_schedule_split()
838 		other_data_bytes = bytecount - first_data_bytes;  in dwc2_uframe_schedule_split()
844 		 * - INT transfers never get split in two.  in dwc2_uframe_schedule_split()
845 		 * - ISOC transfers can always transfer 188 bytes the first  in dwc2_uframe_schedule_split()
855 		if (!qh->ep_is_in &&  in dwc2_uframe_schedule_split()
860 			if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
868 		qh->num_hs_transfers = 1 + DIV_ROUND_UP(other_data_bytes, 188);  in dwc2_uframe_schedule_split()
875 		if (qh->ep_type == USB_ENDPOINT_XFER_INT) {  in dwc2_uframe_schedule_split()
877 				qh->num_hs_transfers += 2;  in dwc2_uframe_schedule_split()
879 				qh->num_hs_transfers += 3;  in dwc2_uframe_schedule_split()
881 			if (qh->ep_is_in) {  in dwc2_uframe_schedule_split()
884 				 * Allocate full data bytes for all data.  in dwc2_uframe_schedule_split()
900 			if (qh->ep_is_in) {  in dwc2_uframe_schedule_split()
904 				qh->num_hs_transfers++;  in dwc2_uframe_schedule_split()
907 				last = rel_uframe + qh->num_hs_transfers + 1;  in dwc2_uframe_schedule_split()
911 					qh->num_hs_transfers += 2;  in dwc2_uframe_schedule_split()
913 					qh->num_hs_transfers += 1;  in dwc2_uframe_schedule_split()
917 					qh->num_hs_transfers--;  in dwc2_uframe_schedule_split()
933 		qh->hs_transfers[0].duration_us = HS_USECS_ISO(first_count);  in dwc2_uframe_schedule_split()
934 		for (i = 1; i < qh->num_hs_transfers - 1; i++)  in dwc2_uframe_schedule_split()
935 			qh->hs_transfers[i].duration_us =  in dwc2_uframe_schedule_split()
937 		if (qh->num_hs_transfers > 1)  in dwc2_uframe_schedule_split()
938 			qh->hs_transfers[qh->num_hs_transfers - 1].duration_us =  in dwc2_uframe_schedule_split()
946 		qh->hs_transfers[0].start_schedule_us =  in dwc2_uframe_schedule_split()
948 		for (i = 1; i < qh->num_hs_transfers; i++)  in dwc2_uframe_schedule_split()
949 			qh->hs_transfers[i].start_schedule_us =  in dwc2_uframe_schedule_split()
950 				((second_s_uframe + i - 1) %  in dwc2_uframe_schedule_split()
955 		for (i = 0; i < qh->num_hs_transfers; i++) {  in dwc2_uframe_schedule_split()
962 		if (i == qh->num_hs_transfers)  in dwc2_uframe_schedule_split()
965 		for (; i >= 0; i--)  in dwc2_uframe_schedule_split()
968 		if (qh->schedule_low_speed)  in dwc2_uframe_schedule_split()
976 		return -ENOSPC;  in dwc2_uframe_schedule_split()
982  * dwc2_uframe_schedule_hs - Schedule a QH for a periodic high speed xfer.
992 	/* In non-split host and device time are the same */  in dwc2_uframe_schedule_hs()
993 	WARN_ON(qh->host_us != qh->device_us);  in dwc2_uframe_schedule_hs()
994 	WARN_ON(qh->host_interval != qh->device_interval);  in dwc2_uframe_schedule_hs()
995 	WARN_ON(qh->num_hs_transfers != 1);  in dwc2_uframe_schedule_hs()
998 	qh->hs_transfers[0].start_schedule_us = 0;  in dwc2_uframe_schedule_hs()
999 	qh->hs_transfers[0].duration_us = qh->host_us;  in dwc2_uframe_schedule_hs()
1005  * dwc2_uframe_schedule_ls - Schedule a QH for a periodic low/full speed xfer.
1015 	/* In non-split host and device time are the same */  in dwc2_uframe_schedule_ls()
1016 	WARN_ON(qh->host_us != qh->device_us);  in dwc2_uframe_schedule_ls()
1017 	WARN_ON(qh->host_interval != qh->device_interval);  in dwc2_uframe_schedule_ls()
1018 	WARN_ON(!qh->schedule_low_speed);  in dwc2_uframe_schedule_ls()
1025  * dwc2_uframe_schedule - Schedule a QH for a periodic xfer.
1027  * Calls one of the 3 sub-function depending on what type of transfer this QH
1037 	if (qh->dev_speed == USB_SPEED_HIGH)  in dwc2_uframe_schedule()
1039 	else if (!qh->do_split)  in dwc2_uframe_schedule()
1053  * dwc2_uframe_unschedule - Undoes dwc2_uframe_schedule().
1062 	for (i = 0; i < qh->num_hs_transfers; i++)  in dwc2_uframe_unschedule()
1065 	if (qh->schedule_low_speed)  in dwc2_uframe_unschedule()
1072  * dwc2_pick_first_frame() - Choose 1st frame for qh that's already scheduled
1079  * running.  It just picks the next frame that we can fit into without any
1095 	 * Use the real frame number rather than the cached value as of the  in dwc2_pick_first_frame()
1101 	 * We wouldn't want to start any earlier than the next frame just in  in dwc2_pick_first_frame()
1102 	 * case the frame number ticks as we're doing this calculation.  in dwc2_pick_first_frame()
1112 	if (!hsotg->params.uframe_sched) {  in dwc2_pick_first_frame()
1113 		if (qh->do_split)  in dwc2_pick_first_frame()
1119 	if (qh->dev_speed == USB_SPEED_HIGH || qh->do_split) {  in dwc2_pick_first_frame()
1123 		 * the first frame should be based on when the first scheduled  in dwc2_pick_first_frame()
1126 		WARN_ON(qh->num_hs_transfers < 1);  in dwc2_pick_first_frame()
1128 		relative_frame = qh->hs_transfers[0].start_schedule_us /  in dwc2_pick_first_frame()
1132 		interval = gcd(qh->host_interval, DWC2_HS_SCHEDULE_UFRAMES);  in dwc2_pick_first_frame()
1136 		 * Low or full speed directly on dwc2.  Just about the same  in dwc2_pick_first_frame()
1142 		relative_frame = qh->ls_start_schedule_slice /  in dwc2_pick_first_frame()
1144 		interval = gcd(qh->host_interval, DWC2_LS_SCHEDULE_FRAMES);  in dwc2_pick_first_frame()
1147 	/* Scheduler messed up if frame is past interval */  in dwc2_pick_first_frame()
1161 	 * Actually choose to start at the frame number we've been  in dwc2_pick_first_frame()
1168 	 * We actually need 1 frame before since the next_active_frame is  in dwc2_pick_first_frame()
1169 	 * the frame number we'll be put on the ready list and we won't be on  in dwc2_pick_first_frame()
1170 	 * the bus until 1 frame later.  in dwc2_pick_first_frame()
1183 	qh->next_active_frame = next_active_frame;  in dwc2_pick_first_frame()
1184 	qh->start_active_frame = next_active_frame;  in dwc2_pick_first_frame()
1187 		      qh, frame_number, qh->next_active_frame);  in dwc2_pick_first_frame()
1191  * dwc2_do_reserve() - Make a periodic reservation
1205 	if (hsotg->params.uframe_sched) {  in dwc2_do_reserve()
1210 			dev_info(hsotg->dev,  in dwc2_do_reserve()
1220 		dev_dbg(hsotg->dev,  in dwc2_do_reserve()
1226 	if (!hsotg->params.uframe_sched)  in dwc2_do_reserve()
1228 		hsotg->periodic_channels++;  in dwc2_do_reserve()
1230 	/* Update claimed usecs per (micro)frame */  in dwc2_do_reserve()
1231 	hsotg->periodic_usecs += qh->host_us;  in dwc2_do_reserve()
1239  * dwc2_do_unreserve() - Actually release the periodic reservation
1249 	assert_spin_locked(&hsotg->lock);  in dwc2_do_unreserve()
1251 	WARN_ON(!qh->unreserve_pending);  in dwc2_do_unreserve()
1253 	/* No more unreserve pending--we're doing it */  in dwc2_do_unreserve()
1254 	qh->unreserve_pending = false;  in dwc2_do_unreserve()
1256 	if (WARN_ON(!list_empty(&qh->qh_list_entry)))  in dwc2_do_unreserve()
1257 		list_del_init(&qh->qh_list_entry);  in dwc2_do_unreserve()
1259 	/* Update claimed usecs per (micro)frame */  in dwc2_do_unreserve()
1260 	hsotg->periodic_usecs -= qh->host_us;  in dwc2_do_unreserve()
1262 	if (hsotg->params.uframe_sched) {  in dwc2_do_unreserve()
1266 		hsotg->periodic_channels--;  in dwc2_do_unreserve()
1271  * dwc2_unreserve_timer_fn() - Timer function to release periodic reservation
1285 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_unreserve_timer_fn()
1291 	 * - We started executing but didn't get the lock yet.  in dwc2_unreserve_timer_fn()
1292 	 * - A new reservation came in, but cancel didn't take effect  in dwc2_unreserve_timer_fn()
1294 	 * - The timer has been kicked again.  in dwc2_unreserve_timer_fn()
1297 	while (!spin_trylock_irqsave(&hsotg->lock, flags)) {  in dwc2_unreserve_timer_fn()
1298 		if (timer_pending(&qh->unreserve_timer))  in dwc2_unreserve_timer_fn()
1304 	 * - We started executing but didn't get the lock yet.  in dwc2_unreserve_timer_fn()
1305 	 * - A new reservation came in, but cancel didn't take effect  in dwc2_unreserve_timer_fn()
1312 	if (qh->unreserve_pending)  in dwc2_unreserve_timer_fn()
1315 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_unreserve_timer_fn()
1319  * dwc2_check_max_xfer_size() - Checks that the max transfer size allowed in a
1321  * (micro)frame for a periodic transfer
1335 	max_xfer_size = qh->maxp * qh->maxp_mult;  in dwc2_check_max_xfer_size()
1336 	max_channel_xfer_size = hsotg->params.max_transfer_size;  in dwc2_check_max_xfer_size()
1339 		dev_err(hsotg->dev,  in dwc2_check_max_xfer_size()
1342 		status = -ENOSPC;  in dwc2_check_max_xfer_size()
1349  * dwc2_schedule_periodic() - Schedules an interrupt or isochronous transfer in
1364 		dev_dbg(hsotg->dev,  in dwc2_schedule_periodic()
1371 	if (del_timer(&qh->unreserve_timer))  in dwc2_schedule_periodic()
1372 		WARN_ON(!qh->unreserve_pending);  in dwc2_schedule_periodic()
1381 	if (!qh->unreserve_pending) {  in dwc2_schedule_periodic()
1392 		if (dwc2_frame_num_le(qh->next_active_frame,  in dwc2_schedule_periodic()
1393 				      hsotg->frame_number))  in dwc2_schedule_periodic()
1397 	qh->unreserve_pending = 0;  in dwc2_schedule_periodic()
1399 	if (hsotg->params.dma_desc_enable)  in dwc2_schedule_periodic()
1401 		list_add_tail(&qh->qh_list_entry, &hsotg->periodic_sched_ready);  in dwc2_schedule_periodic()
1404 		list_add_tail(&qh->qh_list_entry,  in dwc2_schedule_periodic()
1405 			      &hsotg->periodic_sched_inactive);  in dwc2_schedule_periodic()
1411  * dwc2_deschedule_periodic() - Removes an interrupt or isochronous transfer
1422 	assert_spin_locked(&hsotg->lock);  in dwc2_deschedule_periodic()
1426 	 * - Unreserve worker might be sitting there waiting to grab the lock.  in dwc2_deschedule_periodic()
1429 	 * - Unreserve worker might not be scheduled.  in dwc2_deschedule_periodic()
1439 	did_modify = mod_timer(&qh->unreserve_timer,  in dwc2_deschedule_periodic()
1442 	qh->unreserve_pending = 1;  in dwc2_deschedule_periodic()
1444 	list_del_init(&qh->qh_list_entry);  in dwc2_deschedule_periodic()
1448  * dwc2_wait_timer_fn() - Timer function to re-queue after waiting
1473 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_wait_timer_fn()
1476 	spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1482 	if (!qh->wait_timer_cancel) {  in dwc2_wait_timer_fn()
1485 		qh->want_wait = false;  in dwc2_wait_timer_fn()
1487 		list_move(&qh->qh_list_entry,  in dwc2_wait_timer_fn()
1488 			  &hsotg->non_periodic_sched_inactive);  in dwc2_wait_timer_fn()
1495 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1500  * dwc2_qh_init() - Initializes a QH structure
1511 	int dev_speed = dwc2_host_get_speed(hsotg, urb->priv);  in dwc2_qh_init()
1512 	u8 ep_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);  in dwc2_qh_init()
1513 	bool ep_is_in = !!dwc2_hcd_is_pipe_in(&urb->pipe_info);  in dwc2_qh_init()
1520 	int maxp = dwc2_hcd_get_maxp(&urb->pipe_info);  in dwc2_qh_init()
1521 	int maxp_mult = dwc2_hcd_get_maxp_mult(&urb->pipe_info);  in dwc2_qh_init()
1526 	qh->hsotg = hsotg;  in dwc2_qh_init()
1527 	timer_setup(&qh->unreserve_timer, dwc2_unreserve_timer_fn, 0);  in dwc2_qh_init()
1528 	hrtimer_init(&qh->wait_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);  in dwc2_qh_init()
1529 	qh->wait_timer.function = &dwc2_wait_timer_fn;  in dwc2_qh_init()
1530 	qh->ep_type = ep_type;  in dwc2_qh_init()
1531 	qh->ep_is_in = ep_is_in;  in dwc2_qh_init()
1533 	qh->data_toggle = DWC2_HC_PID_DATA0;  in dwc2_qh_init()
1534 	qh->maxp = maxp;  in dwc2_qh_init()
1535 	qh->maxp_mult = maxp_mult;  in dwc2_qh_init()
1536 	INIT_LIST_HEAD(&qh->qtd_list);  in dwc2_qh_init()
1537 	INIT_LIST_HEAD(&qh->qh_list_entry);  in dwc2_qh_init()
1539 	qh->do_split = do_split;  in dwc2_qh_init()
1540 	qh->dev_speed = dev_speed;  in dwc2_qh_init()
1545 		struct dwc2_tt *dwc_tt = dwc2_host_get_tt_info(hsotg, urb->priv,  in dwc2_qh_init()
1547 							       &qh->ttport);  in dwc2_qh_init()
1550 		qh->dwc_tt = dwc_tt;  in dwc2_qh_init()
1552 		qh->host_us = NS_TO_US(usb_calc_bus_time(host_speed, ep_is_in,  in dwc2_qh_init()
1558 			device_ns += dwc_tt->usb_tt->think_time;  in dwc2_qh_init()
1559 		qh->device_us = NS_TO_US(device_ns);  in dwc2_qh_init()
1561 		qh->device_interval = urb->interval;  in dwc2_qh_init()
1562 		qh->host_interval = urb->interval * (do_split ? 8 : 1);  in dwc2_qh_init()
1566 		 * full speed OR if we've got a "TT" to deal with to access this  in dwc2_qh_init()
1569 		qh->schedule_low_speed = prtspd != HPRT0_SPD_HIGH_SPEED ||  in dwc2_qh_init()
1574 			qh->num_hs_transfers = -1;  in dwc2_qh_init()
1576 			qh->num_hs_transfers = 1;  in dwc2_qh_init()
1578 			qh->num_hs_transfers = 0;  in dwc2_qh_init()
1589 		speed = "full";  in dwc2_qh_init()
1599 	switch (qh->ep_type) {  in dwc2_qh_init()
1620 		     dwc2_hcd_get_dev_addr(&urb->pipe_info),  in dwc2_qh_init()
1621 		     dwc2_hcd_get_ep_num(&urb->pipe_info),  in dwc2_qh_init()
1626 			     qh, qh->host_us, qh->device_us);  in dwc2_qh_init()
1628 			     qh, qh->host_interval, qh->device_interval);  in dwc2_qh_init()
1629 		if (qh->schedule_low_speed)  in dwc2_qh_init()
1636  * dwc2_hcd_qh_create() - Allocates and initializes a QH
1651 	if (!urb->priv)  in dwc2_hcd_qh_create()
1661 	if (hsotg->params.dma_desc_enable &&  in dwc2_hcd_qh_create()
1671  * dwc2_hcd_qh_free() - Frees the QH
1684 	if (del_timer_sync(&qh->unreserve_timer)) {  in dwc2_hcd_qh_free()
1687 		spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1689 		spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1699 	hrtimer_cancel(&qh->wait_timer);  in dwc2_hcd_qh_free()
1701 	dwc2_host_put_tt_info(hsotg, qh->dwc_tt);  in dwc2_hcd_qh_free()
1703 	if (qh->desc_list)  in dwc2_hcd_qh_free()
1705 	else if (hsotg->unaligned_cache && qh->dw_align_buf)  in dwc2_hcd_qh_free()
1706 		kmem_cache_free(hsotg->unaligned_cache, qh->dw_align_buf);  in dwc2_hcd_qh_free()
1712  * dwc2_hcd_qh_add() - Adds a QH to either the non periodic or periodic
1728 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_add()
1730 	if (!list_empty(&qh->qh_list_entry))  in dwc2_hcd_qh_add()
1737 		qh->start_active_frame = hsotg->frame_number;  in dwc2_hcd_qh_add()
1738 		qh->next_active_frame = qh->start_active_frame;  in dwc2_hcd_qh_add()
1740 		if (qh->want_wait) {  in dwc2_hcd_qh_add()
1741 			list_add_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_add()
1742 				      &hsotg->non_periodic_sched_waiting);  in dwc2_hcd_qh_add()
1743 			qh->wait_timer_cancel = false;  in dwc2_hcd_qh_add()
1745 			hrtimer_start(&qh->wait_timer, delay, HRTIMER_MODE_REL);  in dwc2_hcd_qh_add()
1747 			list_add_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_add()
1748 				      &hsotg->non_periodic_sched_inactive);  in dwc2_hcd_qh_add()
1756 	if (!hsotg->periodic_qh_count) {  in dwc2_hcd_qh_add()
1761 	hsotg->periodic_qh_count++;  in dwc2_hcd_qh_add()
1767  * dwc2_hcd_qh_unlink() - Removes a QH from either the non-periodic or periodic
1777 	dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_unlink()
1780 	qh->wait_timer_cancel = true;  in dwc2_hcd_qh_unlink()
1782 	if (list_empty(&qh->qh_list_entry))  in dwc2_hcd_qh_unlink()
1787 		if (hsotg->non_periodic_qh_ptr == &qh->qh_list_entry)  in dwc2_hcd_qh_unlink()
1788 			hsotg->non_periodic_qh_ptr =  in dwc2_hcd_qh_unlink()
1789 					hsotg->non_periodic_qh_ptr->next;  in dwc2_hcd_qh_unlink()
1790 		list_del_init(&qh->qh_list_entry);  in dwc2_hcd_qh_unlink()
1795 	hsotg->periodic_qh_count--;  in dwc2_hcd_qh_unlink()
1796 	if (!hsotg->periodic_qh_count &&  in dwc2_hcd_qh_unlink()
1797 	    !hsotg->params.dma_desc_enable) {  in dwc2_hcd_qh_unlink()
1805  * dwc2_next_for_periodic_split() - Set next_active_frame midway thru a split.
1810  * Periodic splits are single low/full speed transfers that we end up splitting
1811  * up into several high speed transfers.  They always fit into one full (1 ms)
1812  * frame but might be split over several microframes (125 us each).  We to put
1813  * each of the parts on a very specific high speed frame.
1819  * @frame_number: The current frame number.
1826 	u16 old_frame = qh->next_active_frame;  in dwc2_next_for_periodic_split()
1837 	if (old_frame == qh->start_active_frame &&  in dwc2_next_for_periodic_split()
1838 	    !(qh->ep_type == USB_ENDPOINT_XFER_ISOC && !qh->ep_is_in))  in dwc2_next_for_periodic_split()
1843 	qh->next_active_frame = dwc2_frame_num_inc(old_frame, incr);  in dwc2_next_for_periodic_split()
1846 	 * Note that it's OK for frame_number to be 1 frame past  in dwc2_next_for_periodic_split()
1848 	 * be 1 frame _before_ when we want to be scheduled.  If we're 1 frame  in dwc2_next_for_periodic_split()
1851 	 * It's _not_ OK, however, if we're more than one frame past.  in dwc2_next_for_periodic_split()
1853 	if (dwc2_frame_num_gt(prev_frame_number, qh->next_active_frame)) {  in dwc2_next_for_periodic_split()
1859 					    qh->next_active_frame);  in dwc2_next_for_periodic_split()
1860 		qh->next_active_frame = frame_number;  in dwc2_next_for_periodic_split()
1867  * dwc2_next_periodic_start() - Set next_active_frame for next transfer start
1882  * @frame_number: The current frame number.
1890 	u16 interval = qh->host_interval;  in dwc2_next_periodic_start()
1893 	qh->start_active_frame = dwc2_frame_num_inc(qh->start_active_frame,  in dwc2_next_periodic_start()
1899 	 * frame counter only goes to 0x3fff.  It's terribly unlikely that we  in dwc2_next_periodic_start()
1911 	 * - We compare against prev_frame_number since start_active_frame  in dwc2_next_periodic_start()
1912 	 *   and next_active_frame are always 1 frame before we want things  in dwc2_next_periodic_start()
1914 	 *   current frame number.  in dwc2_next_periodic_start()
1915 	 * - It's possible for start_active_frame (now incremented) to be  in dwc2_next_periodic_start()
1920 	 *   another transfer for the same frame.  My test webcam doesn't seem  in dwc2_next_periodic_start()
1922 	 *   we do two transfers in the same frame.  in dwc2_next_periodic_start()
1923 	 * - Some misses are expected.  Specifically, in order to work  in dwc2_next_periodic_start()
1933 	if (qh->start_active_frame == qh->next_active_frame ||  in dwc2_next_periodic_start()
1934 	    dwc2_frame_num_gt(prev_frame_number, qh->start_active_frame)) {  in dwc2_next_periodic_start()
1935 		u16 ideal_start = qh->start_active_frame;  in dwc2_next_periodic_start()
1942 		if (qh->do_split || qh->dev_speed == USB_SPEED_HIGH)  in dwc2_next_periodic_start()
1949 			qh->start_active_frame = dwc2_frame_num_inc(  in dwc2_next_periodic_start()
1950 				qh->start_active_frame, interval);  in dwc2_next_periodic_start()
1952 					   qh->start_active_frame));  in dwc2_next_periodic_start()
1954 		missed = dwc2_frame_num_dec(qh->start_active_frame,  in dwc2_next_periodic_start()
1959 	qh->next_active_frame = qh->start_active_frame;  in dwc2_next_periodic_start()
1965  * Deactivates a QH. For non-periodic QHs, removes the QH from the active
1966  * non-periodic schedule. The QH is added to the inactive non-periodic
1972  * scheduled frame is calculated. The QH is placed in the ready schedule if
1973  * the scheduled frame has been reached already. Otherwise it's placed in the
1980 	u16 old_frame = qh->next_active_frame;  in dwc2_hcd_qh_deactivate()
1985 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_deactivate()
1989 		if (!list_empty(&qh->qtd_list))  in dwc2_hcd_qh_deactivate()
1990 			/* Add back to inactive/waiting non-periodic schedule */  in dwc2_hcd_qh_deactivate()
1996 	 * Use the real frame number rather than the cached value as of the  in dwc2_hcd_qh_deactivate()
1999 	 * interrupt for this frame.  in dwc2_hcd_qh_deactivate()
2011 		     qh->next_active_frame,  in dwc2_hcd_qh_deactivate()
2012 		     dwc2_frame_num_dec(qh->next_active_frame, old_frame),  in dwc2_hcd_qh_deactivate()
2015 	if (list_empty(&qh->qtd_list)) {  in dwc2_hcd_qh_deactivate()
2027 	if (dwc2_frame_num_le(qh->next_active_frame, hsotg->frame_number))  in dwc2_hcd_qh_deactivate()
2028 		list_move_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_deactivate()
2029 			       &hsotg->periodic_sched_ready);  in dwc2_hcd_qh_deactivate()
2031 		list_move_tail(&qh->qh_list_entry,  in dwc2_hcd_qh_deactivate()
2032 			       &hsotg->periodic_sched_inactive);  in dwc2_hcd_qh_deactivate()
2036  * dwc2_hcd_qtd_init() - Initializes a QTD structure
2043 	qtd->urb = urb;  in dwc2_hcd_qtd_init()
2044 	if (dwc2_hcd_get_pipe_type(&urb->pipe_info) ==  in dwc2_hcd_qtd_init()
2051 		qtd->data_toggle = DWC2_HC_PID_DATA1;  in dwc2_hcd_qtd_init()
2052 		qtd->control_phase = DWC2_CONTROL_SETUP;  in dwc2_hcd_qtd_init()
2056 	qtd->complete_split = 0;  in dwc2_hcd_qtd_init()
2057 	qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;  in dwc2_hcd_qtd_init()
2058 	qtd->isoc_split_offset = 0;  in dwc2_hcd_qtd_init()
2059 	qtd->in_process = 0;  in dwc2_hcd_qtd_init()
2062 	urb->qtd = qtd;  in dwc2_hcd_qtd_init()
2066  * dwc2_hcd_qtd_add() - Adds a QTD to the QTD-list of a QH
2084 		dev_err(hsotg->dev, "%s: Invalid QH\n", __func__);  in dwc2_hcd_qtd_add()
2085 		retval = -EINVAL;  in dwc2_hcd_qtd_add()
2093 	qtd->qh = qh;  in dwc2_hcd_qtd_add()
2094 	list_add_tail(&qtd->qtd_list_entry, &qh->qtd_list);  in dwc2_hcd_qtd_add()