72 static int dwc2_periodic_channel_available(struct dwc2_hsotg *hsotg)  in dwc2_periodic_channel_available()  argument
82 	num_channels = hsotg->params.host_channels;  in dwc2_periodic_channel_available()
83 	if ((hsotg->periodic_channels + hsotg->non_periodic_channels <  in dwc2_periodic_channel_available()
84 	     num_channels) && (hsotg->periodic_channels < num_channels - 1)) {  in dwc2_periodic_channel_available()
87 		dev_dbg(hsotg->dev,  in dwc2_periodic_channel_available()
90 			hsotg->periodic_channels, hsotg->non_periodic_channels);  in dwc2_periodic_channel_available()
109 static int dwc2_check_periodic_bandwidth(struct dwc2_hsotg *hsotg,  in dwc2_check_periodic_bandwidth()  argument
131 	if (hsotg->periodic_usecs > max_claimed_usecs) {  in dwc2_check_periodic_bandwidth()
132 		dev_err(hsotg->dev,  in dwc2_check_periodic_bandwidth()
134 			__func__, hsotg->periodic_usecs, qh->host_us);  in dwc2_check_periodic_bandwidth()
374 static unsigned long *dwc2_get_ls_map(struct dwc2_hsotg *hsotg,  in dwc2_get_ls_map()  argument
492 	struct dwc2_hsotg *hsotg;  member
506 	dwc2_sch_dbg(print_data->hsotg, "QH=%p ...%s\n", print_data->qh, str);  in dwc2_qh_print()
515 static void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,  in dwc2_qh_schedule_print()  argument
518 	struct dwc2_qh_print_data print_data = { hsotg, qh };  in dwc2_qh_schedule_print()
528 		unsigned long *map = dwc2_get_ls_map(hsotg, qh);  in dwc2_qh_schedule_print()
530 		dwc2_sch_dbg(hsotg, "QH=%p LS/FS trans: %d=>%d us @ %d us",  in dwc2_qh_schedule_print()
536 			dwc2_sch_dbg(hsotg,  in dwc2_qh_schedule_print()
552 		dwc2_sch_dbg(hsotg,  in dwc2_qh_schedule_print()
557 		dwc2_sch_dbg(hsotg, "QH=%p Whole high speed map now:\n", qh);  in dwc2_qh_schedule_print()
558 		pmap_print(hsotg->hs_periodic_bitmap,  in dwc2_qh_schedule_print()
565 static inline void dwc2_qh_schedule_print(struct dwc2_hsotg *hsotg,  in dwc2_qh_schedule_print()  argument
584 static int dwc2_ls_pmap_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_ls_pmap_schedule()  argument
588 	unsigned long *map = dwc2_get_ls_map(hsotg, qh);  in dwc2_ls_pmap_schedule()
623 static void dwc2_ls_pmap_unschedule(struct dwc2_hsotg *hsotg,  in dwc2_ls_pmap_unschedule()  argument
627 	unsigned long *map = dwc2_get_ls_map(hsotg, qh);  in dwc2_ls_pmap_unschedule()
658 static int dwc2_hs_pmap_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_hs_pmap_schedule()  argument
664 	us = pmap_schedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_schedule()
684 static void dwc2_hs_pmap_unschedule(struct dwc2_hsotg *hsotg,  in dwc2_hs_pmap_unschedule()  argument
689 	pmap_unschedule(hsotg->hs_periodic_bitmap,  in dwc2_hs_pmap_unschedule()
708 static int dwc2_uframe_schedule_split(struct dwc2_hsotg *hsotg,  in dwc2_uframe_schedule_split()  argument
749 			err = dwc2_ls_pmap_schedule(hsotg, qh, ls_search_slice);  in dwc2_uframe_schedule_split()
784 				dwc2_ls_pmap_unschedule(hsotg, qh);  in dwc2_uframe_schedule_split()
857 			dwc2_sch_dbg(hsotg,  in dwc2_uframe_schedule_split()
861 				dwc2_ls_pmap_unschedule(hsotg, qh);  in dwc2_uframe_schedule_split()
956 			err = dwc2_hs_pmap_schedule(hsotg, qh, true, i);  in dwc2_uframe_schedule_split()
966 			dwc2_hs_pmap_unschedule(hsotg, qh, i);  in dwc2_uframe_schedule_split()
969 			dwc2_ls_pmap_unschedule(hsotg, qh);  in dwc2_uframe_schedule_split()
990 static int dwc2_uframe_schedule_hs(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_uframe_schedule_hs()  argument
1001 	return dwc2_hs_pmap_schedule(hsotg, qh, false, 0);  in dwc2_uframe_schedule_hs()
1013 static int dwc2_uframe_schedule_ls(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_uframe_schedule_ls()  argument
1021 	return dwc2_ls_pmap_schedule(hsotg, qh, 0);  in dwc2_uframe_schedule_ls()
1033 static int dwc2_uframe_schedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_uframe_schedule()  argument
1038 		ret = dwc2_uframe_schedule_hs(hsotg, qh);  in dwc2_uframe_schedule()
1040 		ret = dwc2_uframe_schedule_ls(hsotg, qh);  in dwc2_uframe_schedule()
1042 		ret = dwc2_uframe_schedule_split(hsotg, qh);  in dwc2_uframe_schedule()
1045 		dwc2_sch_dbg(hsotg, "QH=%p Failed to schedule %d\n", qh, ret);  in dwc2_uframe_schedule()
1047 		dwc2_qh_schedule_print(hsotg, qh);  in dwc2_uframe_schedule()
1058 static void dwc2_uframe_unschedule(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_uframe_unschedule()  argument
1063 		dwc2_hs_pmap_unschedule(hsotg, qh, i);  in dwc2_uframe_unschedule()
1066 		dwc2_ls_pmap_unschedule(hsotg, qh);  in dwc2_uframe_unschedule()
1068 	dwc2_sch_dbg(hsotg, "QH=%p Unscheduled\n", qh);  in dwc2_uframe_unschedule()
1086 static void dwc2_pick_first_frame(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_pick_first_frame()  argument
1098 	frame_number = dwc2_hcd_get_frame_number(hsotg);  in dwc2_pick_first_frame()
1112 	if (!hsotg->params.uframe_sched) {  in dwc2_pick_first_frame()
1186 	dwc2_sch_vdbg(hsotg, "QH=%p First fn=%04x nxt=%04x\n",  in dwc2_pick_first_frame()
1201 static int dwc2_do_reserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_do_reserve()  argument
1205 	if (hsotg->params.uframe_sched) {  in dwc2_do_reserve()
1206 		status = dwc2_uframe_schedule(hsotg, qh);  in dwc2_do_reserve()
1208 		status = dwc2_periodic_channel_available(hsotg);  in dwc2_do_reserve()
1210 			dev_info(hsotg->dev,  in dwc2_do_reserve()
1216 		status = dwc2_check_periodic_bandwidth(hsotg, qh);  in dwc2_do_reserve()
1220 		dev_dbg(hsotg->dev,  in dwc2_do_reserve()
1226 	if (!hsotg->params.uframe_sched)  in dwc2_do_reserve()
1228 		hsotg->periodic_channels++;  in dwc2_do_reserve()
1231 	hsotg->periodic_usecs += qh->host_us;  in dwc2_do_reserve()
1233 	dwc2_pick_first_frame(hsotg, qh);  in dwc2_do_reserve()
1247 static void dwc2_do_unreserve(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_do_unreserve()  argument
1249 	assert_spin_locked(&hsotg->lock);  in dwc2_do_unreserve()
1260 	hsotg->periodic_usecs -= qh->host_us;  in dwc2_do_unreserve()
1262 	if (hsotg->params.uframe_sched) {  in dwc2_do_unreserve()
1263 		dwc2_uframe_unschedule(hsotg, qh);  in dwc2_do_unreserve()
1266 		hsotg->periodic_channels--;  in dwc2_do_unreserve()
1285 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_unreserve_timer_fn()  local
1297 	while (!spin_trylock_irqsave(&hsotg->lock, flags)) {  in dwc2_unreserve_timer_fn()
1313 		dwc2_do_unreserve(hsotg, qh);  in dwc2_unreserve_timer_fn()
1315 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_unreserve_timer_fn()
1328 static int dwc2_check_max_xfer_size(struct dwc2_hsotg *hsotg,  in dwc2_check_max_xfer_size()  argument
1336 	max_channel_xfer_size = hsotg->params.max_transfer_size;  in dwc2_check_max_xfer_size()
1339 		dev_err(hsotg->dev,  in dwc2_check_max_xfer_size()
1358 static int dwc2_schedule_periodic(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_schedule_periodic()  argument
1362 	status = dwc2_check_max_xfer_size(hsotg, qh);  in dwc2_schedule_periodic()
1364 		dev_dbg(hsotg->dev,  in dwc2_schedule_periodic()
1382 		status = dwc2_do_reserve(hsotg, qh);  in dwc2_schedule_periodic()
1393 				      hsotg->frame_number))  in dwc2_schedule_periodic()
1394 			dwc2_pick_first_frame(hsotg, qh);  in dwc2_schedule_periodic()
1399 	if (hsotg->params.dma_desc_enable)  in dwc2_schedule_periodic()
1401 		list_add_tail(&qh->qh_list_entry, &hsotg->periodic_sched_ready);  in dwc2_schedule_periodic()
1405 			      &hsotg->periodic_sched_inactive);  in dwc2_schedule_periodic()
1417 static void dwc2_deschedule_periodic(struct dwc2_hsotg *hsotg,  in dwc2_deschedule_periodic()  argument
1422 	assert_spin_locked(&hsotg->lock);  in dwc2_deschedule_periodic()
1471 	struct dwc2_hsotg *hsotg = qh->hsotg;  in dwc2_wait_timer_fn()  local
1474 	spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1486 			  &hsotg->non_periodic_sched_inactive);  in dwc2_wait_timer_fn()
1488 		tr_type = dwc2_hcd_select_transactions(hsotg);  in dwc2_wait_timer_fn()
1490 			dwc2_hcd_queue_transactions(hsotg, tr_type);  in dwc2_wait_timer_fn()
1493 	spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_wait_timer_fn()
1505 static void dwc2_qh_init(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_qh_init()  argument
1508 	int dev_speed = dwc2_host_get_speed(hsotg, urb->priv);  in dwc2_qh_init()
1513 	u32 hprt = dwc2_readl(hsotg, HPRT0);  in dwc2_qh_init()
1522 	qh->hsotg = hsotg;  in dwc2_qh_init()
1539 		struct dwc2_tt *dwc_tt = dwc2_host_get_tt_info(hsotg, urb->priv,  in dwc2_qh_init()
1611 	dwc2_sch_dbg(hsotg, "QH=%p Init %s, %s speed, %d bytes:\n", qh, type,  in dwc2_qh_init()
1613 	dwc2_sch_dbg(hsotg, "QH=%p ...addr=%d, ep=%d, %s\n", qh,  in dwc2_qh_init()
1618 		dwc2_sch_dbg(hsotg,  in dwc2_qh_init()
1621 		dwc2_sch_dbg(hsotg, "QH=%p ...interval: host=%d, device=%d\n",  in dwc2_qh_init()
1624 			dwc2_sch_dbg(hsotg, "QH=%p ...low speed schedule=%p\n",  in dwc2_qh_init()
1625 				     qh, dwc2_get_ls_map(hsotg, qh));  in dwc2_qh_init()
1639 struct dwc2_qh *dwc2_hcd_qh_create(struct dwc2_hsotg *hsotg,  in dwc2_hcd_qh_create()  argument
1653 	dwc2_qh_init(hsotg, qh, urb, mem_flags);  in dwc2_hcd_qh_create()
1655 	if (hsotg->params.dma_desc_enable &&  in dwc2_hcd_qh_create()
1656 	    dwc2_hcd_qh_init_ddma(hsotg, qh, mem_flags) < 0) {  in dwc2_hcd_qh_create()
1657 		dwc2_hcd_qh_free(hsotg, qh);  in dwc2_hcd_qh_create()
1675 void dwc2_hcd_qh_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_qh_free()  argument
1681 		spin_lock_irqsave(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1682 		dwc2_do_unreserve(hsotg, qh);  in dwc2_hcd_qh_free()
1683 		spin_unlock_irqrestore(&hsotg->lock, flags);  in dwc2_hcd_qh_free()
1695 	dwc2_host_put_tt_info(hsotg, qh->dwc_tt);  in dwc2_hcd_qh_free()
1698 		dwc2_hcd_qh_free_ddma(hsotg, qh);  in dwc2_hcd_qh_free()
1699 	else if (hsotg->unaligned_cache && qh->dw_align_buf)  in dwc2_hcd_qh_free()
1700 		kmem_cache_free(hsotg->unaligned_cache, qh->dw_align_buf);  in dwc2_hcd_qh_free()
1715 int dwc2_hcd_qh_add(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_qh_add()  argument
1721 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_add()
1730 		qh->start_active_frame = hsotg->frame_number;  in dwc2_hcd_qh_add()
1735 				      &hsotg->non_periodic_sched_waiting);  in dwc2_hcd_qh_add()
1741 				      &hsotg->non_periodic_sched_inactive);  in dwc2_hcd_qh_add()
1746 	status = dwc2_schedule_periodic(hsotg, qh);  in dwc2_hcd_qh_add()
1749 	if (!hsotg->periodic_qh_count) {  in dwc2_hcd_qh_add()
1750 		intr_mask = dwc2_readl(hsotg, GINTMSK);  in dwc2_hcd_qh_add()
1752 		dwc2_writel(hsotg, intr_mask, GINTMSK);  in dwc2_hcd_qh_add()
1754 	hsotg->periodic_qh_count++;  in dwc2_hcd_qh_add()
1766 void dwc2_hcd_qh_unlink(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_qh_unlink()  argument
1770 	dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_unlink()
1780 		if (hsotg->non_periodic_qh_ptr == &qh->qh_list_entry)  in dwc2_hcd_qh_unlink()
1781 			hsotg->non_periodic_qh_ptr =  in dwc2_hcd_qh_unlink()
1782 					hsotg->non_periodic_qh_ptr->next;  in dwc2_hcd_qh_unlink()
1787 	dwc2_deschedule_periodic(hsotg, qh);  in dwc2_hcd_qh_unlink()
1788 	hsotg->periodic_qh_count--;  in dwc2_hcd_qh_unlink()
1789 	if (!hsotg->periodic_qh_count &&  in dwc2_hcd_qh_unlink()
1790 	    !hsotg->params.dma_desc_enable) {  in dwc2_hcd_qh_unlink()
1791 		intr_mask = dwc2_readl(hsotg, GINTMSK);  in dwc2_hcd_qh_unlink()
1793 		dwc2_writel(hsotg, intr_mask, GINTMSK);  in dwc2_hcd_qh_unlink()
1816 static int dwc2_next_for_periodic_split(struct dwc2_hsotg *hsotg,  in dwc2_next_for_periodic_split()  argument
1879 static int dwc2_next_periodic_start(struct dwc2_hsotg *hsotg,  in dwc2_next_periodic_start()  argument
1970 void dwc2_hcd_qh_deactivate(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_hcd_qh_deactivate()  argument
1978 		dev_vdbg(hsotg->dev, "%s()\n", __func__);  in dwc2_hcd_qh_deactivate()
1981 		dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_qh_deactivate()
1984 			dwc2_hcd_qh_add(hsotg, qh);  in dwc2_hcd_qh_deactivate()
1994 	frame_number = dwc2_hcd_get_frame_number(hsotg);  in dwc2_hcd_qh_deactivate()
1997 		missed = dwc2_next_for_periodic_split(hsotg, qh, frame_number);  in dwc2_hcd_qh_deactivate()
1999 		missed = dwc2_next_periodic_start(hsotg, qh, frame_number);  in dwc2_hcd_qh_deactivate()
2001 	dwc2_sch_vdbg(hsotg,  in dwc2_hcd_qh_deactivate()
2009 		dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_qh_deactivate()
2020 	if (dwc2_frame_num_le(qh->next_active_frame, hsotg->frame_number))  in dwc2_hcd_qh_deactivate()
2022 			       &hsotg->periodic_sched_ready);  in dwc2_hcd_qh_deactivate()
2025 			       &hsotg->periodic_sched_inactive);  in dwc2_hcd_qh_deactivate()
2071 int dwc2_hcd_qtd_add(struct dwc2_hsotg *hsotg, struct dwc2_qtd *qtd,  in dwc2_hcd_qtd_add()  argument
2077 		dev_err(hsotg->dev, "%s: Invalid QH\n", __func__);  in dwc2_hcd_qtd_add()
2082 	retval = dwc2_hcd_qh_add(hsotg, qh);  in dwc2_hcd_qtd_add()