75 static u16 dwc2_max_desc_num(struct dwc2_qh *qh)  in dwc2_max_desc_num()  argument
77 	return (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&  in dwc2_max_desc_num()
78 		qh->dev_speed == USB_SPEED_HIGH) ?  in dwc2_max_desc_num()
82 static u16 dwc2_frame_incr_val(struct dwc2_qh *qh)  in dwc2_frame_incr_val()  argument
84 	return qh->dev_speed == USB_SPEED_HIGH ?  in dwc2_frame_incr_val()
85 	       (qh->host_interval + 8 - 1) / 8 : qh->host_interval;  in dwc2_frame_incr_val()
88 static int dwc2_desc_list_alloc(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_desc_list_alloc()  argument
93 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&  in dwc2_desc_list_alloc()
94 	    qh->dev_speed == USB_SPEED_HIGH)  in dwc2_desc_list_alloc()
99 	qh->desc_list_sz = sizeof(struct dwc2_dma_desc) *  in dwc2_desc_list_alloc()
100 						dwc2_max_desc_num(qh);  in dwc2_desc_list_alloc()
102 	qh->desc_list = kmem_cache_zalloc(desc_cache, flags | GFP_DMA);  in dwc2_desc_list_alloc()
103 	if (!qh->desc_list)  in dwc2_desc_list_alloc()
106 	qh->desc_list_dma = dma_map_single(hsotg->dev, qh->desc_list,  in dwc2_desc_list_alloc()
107 					   qh->desc_list_sz,  in dwc2_desc_list_alloc()
110 	qh->n_bytes = kcalloc(dwc2_max_desc_num(qh), sizeof(u32), flags);  in dwc2_desc_list_alloc()
111 	if (!qh->n_bytes) {  in dwc2_desc_list_alloc()
112 		dma_unmap_single(hsotg->dev, qh->desc_list_dma,  in dwc2_desc_list_alloc()
113 				 qh->desc_list_sz,  in dwc2_desc_list_alloc()
115 		kmem_cache_free(desc_cache, qh->desc_list);  in dwc2_desc_list_alloc()
116 		qh->desc_list = NULL;  in dwc2_desc_list_alloc()
123 static void dwc2_desc_list_free(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_desc_list_free()  argument
127 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC &&  in dwc2_desc_list_free()
128 	    qh->dev_speed == USB_SPEED_HIGH)  in dwc2_desc_list_free()
133 	if (qh->desc_list) {  in dwc2_desc_list_free()
134 		dma_unmap_single(hsotg->dev, qh->desc_list_dma,  in dwc2_desc_list_free()
135 				 qh->desc_list_sz, DMA_FROM_DEVICE);  in dwc2_desc_list_free()
136 		kmem_cache_free(desc_cache, qh->desc_list);  in dwc2_desc_list_free()
137 		qh->desc_list = NULL;  in dwc2_desc_list_free()
140 	kfree(qh->n_bytes);  in dwc2_desc_list_free()
141 	qh->n_bytes = NULL;  in dwc2_desc_list_free()
230 static void dwc2_update_frame_list(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_update_frame_list()  argument
241 	if (!qh->channel) {  in dwc2_update_frame_list()
242 		dev_err(hsotg->dev, "qh->channel = %p\n", qh->channel);  in dwc2_update_frame_list()
252 	chan = qh->channel;  in dwc2_update_frame_list()
253 	inc = dwc2_frame_incr_val(qh);  in dwc2_update_frame_list()
254 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC)  in dwc2_update_frame_list()
255 		i = dwc2_frame_list_idx(qh->next_active_frame);  in dwc2_update_frame_list()
281 	if (chan->speed == USB_SPEED_HIGH && qh->host_interval) {  in dwc2_update_frame_list()
284 		inc = (8 + qh->host_interval - 1) / qh->host_interval;  in dwc2_update_frame_list()
287 			j = j << qh->host_interval;  in dwc2_update_frame_list()
295 				      struct dwc2_qh *qh)  in dwc2_release_channel_ddma()  argument
297 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_release_channel_ddma()
299 	if (dwc2_qh_is_non_per(qh)) {  in dwc2_release_channel_ddma()
305 		dwc2_update_frame_list(hsotg, qh, 0);  in dwc2_release_channel_ddma()
313 	if (chan->qh) {  in dwc2_release_channel_ddma()
318 		chan->qh = NULL;  in dwc2_release_channel_ddma()
321 	qh->channel = NULL;  in dwc2_release_channel_ddma()
322 	qh->ntd = 0;  in dwc2_release_channel_ddma()
324 	if (qh->desc_list)  in dwc2_release_channel_ddma()
325 		memset(qh->desc_list, 0, sizeof(struct dwc2_dma_desc) *  in dwc2_release_channel_ddma()
326 		       dwc2_max_desc_num(qh));  in dwc2_release_channel_ddma()
342 int dwc2_hcd_qh_init_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,  in dwc2_hcd_qh_init_ddma()  argument
347 	if (qh->do_split) {  in dwc2_hcd_qh_init_ddma()
354 	retval = dwc2_desc_list_alloc(hsotg, qh, mem_flags);  in dwc2_hcd_qh_init_ddma()
358 	if (qh->ep_type == USB_ENDPOINT_XFER_ISOC ||  in dwc2_hcd_qh_init_ddma()
359 	    qh->ep_type == USB_ENDPOINT_XFER_INT) {  in dwc2_hcd_qh_init_ddma()
369 	qh->ntd = 0;  in dwc2_hcd_qh_init_ddma()
373 	dwc2_desc_list_free(hsotg, qh);  in dwc2_hcd_qh_init_ddma()
388 void dwc2_hcd_qh_free_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_qh_free_ddma()  argument
392 	dwc2_desc_list_free(hsotg, qh);  in dwc2_hcd_qh_free_ddma()
402 	if (qh->channel)  in dwc2_hcd_qh_free_ddma()
403 		dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_qh_free_ddma()
406 	if ((qh->ep_type == USB_ENDPOINT_XFER_ISOC ||  in dwc2_hcd_qh_free_ddma()
407 	     qh->ep_type == USB_ENDPOINT_XFER_INT) &&  in dwc2_hcd_qh_free_ddma()
415 static u8 dwc2_frame_to_desc_idx(struct dwc2_qh *qh, u16 frame_idx)  in dwc2_frame_to_desc_idx()  argument
417 	if (qh->dev_speed == USB_SPEED_HIGH)  in dwc2_frame_to_desc_idx()
429 				    struct dwc2_qh *qh, u16 *skip_frames)  in dwc2_calc_starting_frame()  argument
454 	if (qh->dev_speed == USB_SPEED_HIGH) {  in dwc2_calc_starting_frame()
491 					struct dwc2_qh *qh)  in dwc2_recalc_initial_desc_idx()  argument
513 	if (qh->channel) {  in dwc2_recalc_initial_desc_idx()
514 		frame = dwc2_calc_starting_frame(hsotg, qh, &skip_frames);  in dwc2_recalc_initial_desc_idx()
521 			  dwc2_frame_list_idx(qh->next_active_frame) -  in dwc2_recalc_initial_desc_idx()
522 			  fr_idx_tmp) % dwc2_frame_incr_val(qh);  in dwc2_recalc_initial_desc_idx()
525 		qh->next_active_frame = dwc2_calc_starting_frame(hsotg, qh,  in dwc2_recalc_initial_desc_idx()
527 		fr_idx = dwc2_frame_list_idx(qh->next_active_frame);  in dwc2_recalc_initial_desc_idx()
530 	qh->td_first = qh->td_last = dwc2_frame_to_desc_idx(qh, fr_idx);  in dwc2_recalc_initial_desc_idx()
543 					 struct dwc2_qh *qh, u32 max_xfer_size,  in dwc2_fill_host_isoc_dma_desc()  argument
546 	struct dwc2_dma_desc *dma_desc = &qh->desc_list[idx];  in dwc2_fill_host_isoc_dma_desc()
553 		qh->n_bytes[idx] = max_xfer_size;  in dwc2_fill_host_isoc_dma_desc()
555 		qh->n_bytes[idx] = frame_desc->length;  in dwc2_fill_host_isoc_dma_desc()
558 	dma_desc->status = qh->n_bytes[idx] << HOST_DMA_ISOC_NBYTES_SHIFT &  in dwc2_fill_host_isoc_dma_desc()
564 	qh->ntd++;  in dwc2_fill_host_isoc_dma_desc()
574 				   qh->desc_list_dma +  in dwc2_fill_host_isoc_dma_desc()
581 				    struct dwc2_qh *qh, u16 skip_frames)  in dwc2_init_isoc_dma_desc()  argument
589 	idx = qh->td_last;  in dwc2_init_isoc_dma_desc()
590 	inc = qh->host_interval;  in dwc2_init_isoc_dma_desc()
593 	next_idx = dwc2_desclist_idx_inc(qh->td_last, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
606 			qh->td_last = dwc2_desclist_idx_inc(cur_idx, inc,  in dwc2_init_isoc_dma_desc()
607 							    qh->dev_speed);  in dwc2_init_isoc_dma_desc()
608 			idx = qh->td_last;  in dwc2_init_isoc_dma_desc()
612 	if (qh->host_interval) {  in dwc2_init_isoc_dma_desc()
613 		ntd_max = (dwc2_max_desc_num(qh) + qh->host_interval - 1) /  in dwc2_init_isoc_dma_desc()
614 				qh->host_interval;  in dwc2_init_isoc_dma_desc()
615 		if (skip_frames && !qh->channel)  in dwc2_init_isoc_dma_desc()
616 			ntd_max -= skip_frames / qh->host_interval;  in dwc2_init_isoc_dma_desc()
619 	max_xfer_size = qh->dev_speed == USB_SPEED_HIGH ?  in dwc2_init_isoc_dma_desc()
622 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {  in dwc2_init_isoc_dma_desc()
629 		while (qh->ntd < ntd_max && qtd->isoc_frame_index_last <  in dwc2_init_isoc_dma_desc()
631 			dwc2_fill_host_isoc_dma_desc(hsotg, qtd, qh,  in dwc2_init_isoc_dma_desc()
633 			idx = dwc2_desclist_idx_inc(idx, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
640 	qh->td_last = idx;  in dwc2_init_isoc_dma_desc()
644 	if (qh->ntd == ntd_max) {  in dwc2_init_isoc_dma_desc()
645 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
646 		qh->desc_list[idx].status |= HOST_DMA_IOC;  in dwc2_init_isoc_dma_desc()
648 					   qh->desc_list_dma + (idx *  in dwc2_init_isoc_dma_desc()
670 		idx = dwc2_desclist_idx_dec(idx, inc * ((qh->ntd + 1) / 2),  in dwc2_init_isoc_dma_desc()
671 					    qh->dev_speed);  in dwc2_init_isoc_dma_desc()
678 		idx = dwc2_desclist_idx_dec(qh->td_last, inc, qh->dev_speed);  in dwc2_init_isoc_dma_desc()
680 	qh->desc_list[idx].status |= HOST_DMA_IOC;  in dwc2_init_isoc_dma_desc()
682 				   qh->desc_list_dma +  in dwc2_init_isoc_dma_desc()
691 				    struct dwc2_qtd *qtd, struct dwc2_qh *qh,  in dwc2_fill_host_dma_desc()  argument
694 	struct dwc2_dma_desc *dma_desc = &qh->desc_list[n_desc];  in dwc2_fill_host_dma_desc()
715 	qh->n_bytes[n_desc] = len;  in dwc2_fill_host_dma_desc()
717 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL &&  in dwc2_fill_host_dma_desc()
724 				   qh->desc_list_dma +  in dwc2_fill_host_dma_desc()
742 					struct dwc2_qh *qh)  in dwc2_init_non_isoc_dma_desc()  argument
745 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_init_non_isoc_dma_desc()
748 	dev_vdbg(hsotg->dev, "%s(): qh=%p dma=%08lx len=%d\n", __func__, qh,  in dwc2_init_non_isoc_dma_desc()
758 	list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry) {  in dwc2_init_non_isoc_dma_desc()
774 				qh->desc_list[n_desc - 1].status |= HOST_DMA_A;  in dwc2_init_non_isoc_dma_desc()
778 					 &qh->desc_list[n_desc - 1]);  in dwc2_init_non_isoc_dma_desc()
780 							   qh->desc_list_dma +  in dwc2_init_non_isoc_dma_desc()
786 			dwc2_fill_host_dma_desc(hsotg, chan, qtd, qh, n_desc);  in dwc2_init_non_isoc_dma_desc()
789 				 n_desc, &qh->desc_list[n_desc],  in dwc2_init_non_isoc_dma_desc()
790 				 qh->desc_list[n_desc].buf,  in dwc2_init_non_isoc_dma_desc()
791 				 qh->desc_list[n_desc].status);  in dwc2_init_non_isoc_dma_desc()
799 		if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL)  in dwc2_init_non_isoc_dma_desc()
806 		qh->desc_list[n_desc - 1].status |=  in dwc2_init_non_isoc_dma_desc()
809 			 n_desc - 1, &qh->desc_list[n_desc - 1]);  in dwc2_init_non_isoc_dma_desc()
811 					   qh->desc_list_dma + (n_desc - 1) *  in dwc2_init_non_isoc_dma_desc()
816 			qh->desc_list[0].status |= HOST_DMA_A;  in dwc2_init_non_isoc_dma_desc()
818 				 &qh->desc_list[0]);  in dwc2_init_non_isoc_dma_desc()
820 						   qh->desc_list_dma,  in dwc2_init_non_isoc_dma_desc()
847 void dwc2_hcd_start_xfer_ddma(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh)  in dwc2_hcd_start_xfer_ddma()  argument
850 	struct dwc2_host_chan *chan = qh->channel;  in dwc2_hcd_start_xfer_ddma()
856 		dwc2_init_non_isoc_dma_desc(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
860 		dwc2_init_non_isoc_dma_desc(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
861 		dwc2_update_frame_list(hsotg, qh, 1);  in dwc2_hcd_start_xfer_ddma()
865 		if (!qh->ntd)  in dwc2_hcd_start_xfer_ddma()
866 			skip_frames = dwc2_recalc_initial_desc_idx(hsotg, qh);  in dwc2_hcd_start_xfer_ddma()
867 		dwc2_init_isoc_dma_desc(hsotg, qh, skip_frames);  in dwc2_hcd_start_xfer_ddma()
870 			dwc2_update_frame_list(hsotg, qh, 1);  in dwc2_hcd_start_xfer_ddma()
877 			chan->ntd = dwc2_max_desc_num(qh);  in dwc2_hcd_start_xfer_ddma()
895 					struct dwc2_qh *qh, u16 idx)  in dwc2_cmpl_host_isoc_dma_desc()  argument
905 	dma_sync_single_for_cpu(hsotg->dev, qh->desc_list_dma + (idx *  in dwc2_cmpl_host_isoc_dma_desc()
910 	dma_desc = &qh->desc_list[idx];  in dwc2_cmpl_host_isoc_dma_desc()
925 		frame_desc->actual_length = qh->n_bytes[idx] - remain;  in dwc2_cmpl_host_isoc_dma_desc()
929 		frame_desc->actual_length = qh->n_bytes[idx] - remain;  in dwc2_cmpl_host_isoc_dma_desc()
939 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_cmpl_host_isoc_dma_desc()
952 	qh->ntd--;  in dwc2_cmpl_host_isoc_dma_desc()
967 	struct dwc2_qh *qh;  in dwc2_complete_isoc_xfer_ddma()  local
971 	qh = chan->qh;  in dwc2_complete_isoc_xfer_ddma()
972 	idx = qh->td_first;  in dwc2_complete_isoc_xfer_ddma()
975 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)  in dwc2_complete_isoc_xfer_ddma()
994 		list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list,  in dwc2_complete_isoc_xfer_ddma()
1006 			dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_complete_isoc_xfer_ddma()
1012 	list_for_each_entry_safe(qtd, qtd_tmp, &qh->qtd_list, qtd_list_entry) {  in dwc2_complete_isoc_xfer_ddma()
1032 			rc = dwc2_cmpl_host_isoc_dma_desc(hsotg, chan, qtd, qh,  in dwc2_complete_isoc_xfer_ddma()
1036 			idx = dwc2_desclist_idx_inc(idx, qh->host_interval,  in dwc2_complete_isoc_xfer_ddma()
1046 			if (qh->host_interval >= 32)  in dwc2_complete_isoc_xfer_ddma()
1049 			qh->td_first = idx;  in dwc2_complete_isoc_xfer_ddma()
1051 			qtd_next = list_first_entry(&qh->qtd_list,  in dwc2_complete_isoc_xfer_ddma()
1060 		} while (idx != qh->td_first);  in dwc2_complete_isoc_xfer_ddma()
1064 	qh->td_first = idx;  in dwc2_complete_isoc_xfer_ddma()
1156 	struct dwc2_qh *qh = chan->qh;  in dwc2_process_non_isoc_desc()  local
1168 				qh->desc_list_dma + (desc_num *  in dwc2_process_non_isoc_desc()
1173 	dma_desc = &qh->desc_list[desc_num];  in dwc2_process_non_isoc_desc()
1174 	n_bytes = qh->n_bytes[desc_num];  in dwc2_process_non_isoc_desc()
1183 		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);  in dwc2_process_non_isoc_desc()
1189 	if (qh->ep_type == USB_ENDPOINT_XFER_CONTROL) {  in dwc2_process_non_isoc_desc()
1227 	struct dwc2_qh *qh = chan->qh;  in dwc2_complete_non_isoc_xfer_ddma()  local
1233 		list_for_each_entry(qtd, &qh->qtd_list, qtd_list_entry)  in dwc2_complete_non_isoc_xfer_ddma()
1238 	list_for_each_safe(qtd_item, qtd_tmp, &qh->qtd_list) {  in dwc2_complete_non_isoc_xfer_ddma()
1259 	if (qh->ep_type != USB_ENDPOINT_XFER_CONTROL) {  in dwc2_complete_non_isoc_xfer_ddma()
1265 			qh->data_toggle = DWC2_HC_PID_DATA0;  in dwc2_complete_non_isoc_xfer_ddma()
1277 			qh->ping_state = 1;  in dwc2_complete_non_isoc_xfer_ddma()
1303 	struct dwc2_qh *qh = chan->qh;  in dwc2_hcd_complete_xfer_ddma()  local
1312 		    list_empty(&qh->qtd_list)) {  in dwc2_hcd_complete_xfer_ddma()
1320 						 &qh->qtd_list,  in dwc2_hcd_complete_xfer_ddma()
1325 							     qtd, qh);  in dwc2_hcd_complete_xfer_ddma()
1331 			dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1332 			dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1335 			list_move_tail(&qh->qh_list_entry,  in dwc2_hcd_complete_xfer_ddma()
1355 		dwc2_release_channel_ddma(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1356 		dwc2_hcd_qh_unlink(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()
1358 		if (!list_empty(&qh->qtd_list)) {  in dwc2_hcd_complete_xfer_ddma()
1363 			dwc2_hcd_qh_add(hsotg, qh);  in dwc2_hcd_complete_xfer_ddma()