521 	struct xhci_ring *ep_ring;  in xhci_find_new_dequeue_state()  local
529 	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,  in xhci_find_new_dequeue_state()
531 	if (!ep_ring) {  in xhci_find_new_dequeue_state()
542 	new_seg = ep_ring->deq_seg;  in xhci_find_new_dequeue_state()
543 	new_deq = ep_ring->dequeue;  in xhci_find_new_dequeue_state()
567 		next_trb(xhci, ep_ring, &new_seg, &new_deq);  in xhci_find_new_dequeue_state()
599 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in td_to_noop()  argument
615 		next_trb(xhci, ep_ring, &seg, &trb);  in td_to_noop()
692 	struct xhci_ring *ep_ring;  in xhci_handle_cmd_stop_ep()  local
736 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);  in xhci_handle_cmd_stop_ep()
737 		if (!ep_ring) {  in xhci_handle_cmd_stop_ep()
769 			td_to_noop(xhci, ep_ring, cur_td, false);  in xhci_handle_cmd_stop_ep()
808 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);  in xhci_handle_cmd_stop_ep()
809 		xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);  in xhci_handle_cmd_stop_ep()
973 		struct xhci_ring *ep_ring,  in update_ring_for_set_deq_completion()  argument
980 	num_trbs_free_temp = ep_ring->num_trbs_free;  in update_ring_for_set_deq_completion()
981 	dequeue_temp = ep_ring->dequeue;  in update_ring_for_set_deq_completion()
989 	if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
990 		ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
991 		ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
994 	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {  in update_ring_for_set_deq_completion()
996 		ep_ring->num_trbs_free++;  in update_ring_for_set_deq_completion()
997 		ep_ring->dequeue++;  in update_ring_for_set_deq_completion()
998 		if (trb_is_link(ep_ring->dequeue)) {  in update_ring_for_set_deq_completion()
999 			if (ep_ring->dequeue ==  in update_ring_for_set_deq_completion()
1002 			ep_ring->deq_seg = ep_ring->deq_seg->next;  in update_ring_for_set_deq_completion()
1003 			ep_ring->dequeue = ep_ring->deq_seg->trbs;  in update_ring_for_set_deq_completion()
1005 		if (ep_ring->dequeue == dequeue_temp) {  in update_ring_for_set_deq_completion()
1013 		ep_ring->num_trbs_free = num_trbs_free_temp;  in update_ring_for_set_deq_completion()
1029 	struct xhci_ring *ep_ring;  in xhci_handle_cmd_set_deq()  local
1040 	ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);  in xhci_handle_cmd_set_deq()
1041 	if (!ep_ring) {  in xhci_handle_cmd_set_deq()
1103 				ep_ring, ep_index);  in xhci_handle_cmd_set_deq()
1804 		struct xhci_ring *ep_ring, int *status)  in xhci_td_cleanup()  argument
1812 	xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);  in xhci_td_cleanup()
1855 	struct xhci_ring *ep_ring;  in finish_td()  local
1863 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in finish_td()
1885 					ep_ring->stream_id, td, EP_HARD_RESET);  in finish_td()
1888 		while (ep_ring->dequeue != td->last_trb)  in finish_td()
1889 			inc_deq(xhci, ep_ring);  in finish_td()
1890 		inc_deq(xhci, ep_ring);  in finish_td()
1893 	return xhci_td_cleanup(xhci, td, ep_ring, status);  in finish_td()
2020 	struct xhci_ring *ep_ring;  in process_isoc_td()  local
2029 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in process_isoc_td()
2090 		frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +  in process_isoc_td()
2104 	struct xhci_ring *ep_ring;  in skip_isoc_td()  local
2109 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in skip_isoc_td()
2121 	while (ep_ring->dequeue != td->last_trb)  in skip_isoc_td()
2122 		inc_deq(xhci, ep_ring);  in skip_isoc_td()
2123 	inc_deq(xhci, ep_ring);  in skip_isoc_td()
2125 	return xhci_td_cleanup(xhci, td, ep_ring, status);  in skip_isoc_td()
2135 	struct xhci_ring *ep_ring;  in process_bulk_intr_td()  local
2139 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));  in process_bulk_intr_td()
2179 			sum_trb_lengths(xhci, ep_ring, ep_trb) +  in process_bulk_intr_td()
2200 	struct xhci_ring *ep_ring;  in handle_tx_event()  local
2227 	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);  in handle_tx_event()
2238 	if (!ep_ring) {  in handle_tx_event()
2259 		list_for_each(tmp, &ep_ring->td_list)  in handle_tx_event()
2343 		if (!list_empty(&ep_ring->td_list))  in handle_tx_event()
2351 		if (!list_empty(&ep_ring->td_list))  in handle_tx_event()
2398 		if (list_empty(&ep_ring->td_list)) {  in handle_tx_event()
2409 			      ep_ring->last_td_was_short)) {  in handle_tx_event()
2430 		td = list_first_entry(&ep_ring->td_list, struct xhci_td,  in handle_tx_event()
2436 		ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,  in handle_tx_event()
2460 						ep_ring->last_td_was_short) {  in handle_tx_event()
2461 					ep_ring->last_td_was_short = false;  in handle_tx_event()
2470 				trb_in_td(xhci, ep_ring->deq_seg,  in handle_tx_event()
2471 					  ep_ring->dequeue, td->last_trb,  in handle_tx_event()
2480 			ep_ring->last_td_was_short = true;  in handle_tx_event()
2482 			ep_ring->last_td_was_short = false;  in handle_tx_event()
2494 		trace_xhci_handle_transfer(ep_ring,  in handle_tx_event()
2510 							     ep_ring->stream_id,  in handle_tx_event()
2761 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,  in prepare_ring()  argument
2795 		if (room_on_ring(xhci, ep_ring, num_trbs))  in prepare_ring()
2798 		if (ep_ring == xhci->cmd_ring) {  in prepare_ring()
2805 		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;  in prepare_ring()
2806 		if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,  in prepare_ring()
2813 	while (trb_is_link(ep_ring->enqueue)) {  in prepare_ring()
2818 		    !(ep_ring->type == TYPE_ISOC &&  in prepare_ring()
2820 			ep_ring->enqueue->link.control &=  in prepare_ring()
2823 			ep_ring->enqueue->link.control |=  in prepare_ring()
2827 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);  in prepare_ring()
2830 		if (link_trb_toggles_cycle(ep_ring->enqueue))  in prepare_ring()
2831 			ep_ring->cycle_state ^= 1;  in prepare_ring()
2833 		ep_ring->enq_seg = ep_ring->enq_seg->next;  in prepare_ring()
2834 		ep_ring->enqueue = ep_ring->enq_seg->trbs;  in prepare_ring()
2851 	struct xhci_ring *ep_ring;  in prepare_transfer()  local
2854 	ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);  in prepare_transfer()
2855 	if (!ep_ring) {  in prepare_transfer()
2861 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in prepare_transfer()
2880 	list_add_tail(&td->td_list, &ep_ring->td_list);  in prepare_transfer()
2881 	td->start_seg = ep_ring->enq_seg;  in prepare_transfer()
2882 	td->first_trb = ep_ring->enqueue;  in prepare_transfer()
3271 	struct xhci_ring *ep_ring;  in xhci_queue_ctrl_tx()  local
3281 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_queue_ctrl_tx()
3282 	if (!ep_ring)  in xhci_queue_ctrl_tx()
3315 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_ctrl_tx()
3316 	start_cycle = ep_ring->cycle_state;  in xhci_queue_ctrl_tx()
3336 	queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3362 		queue_trb(xhci, ep_ring, true,  in xhci_queue_ctrl_tx()
3366 				field | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
3370 	td->last_trb = ep_ring->enqueue;  in xhci_queue_ctrl_tx()
3378 	queue_trb(xhci, ep_ring, false,  in xhci_queue_ctrl_tx()
3383 			field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);  in xhci_queue_ctrl_tx()
3538 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx()  local
3554 	ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;  in xhci_queue_isoc_tx()
3562 	start_trb = &ep_ring->enqueue->generic;  in xhci_queue_isoc_tx()
3563 	start_cycle = ep_ring->cycle_state;  in xhci_queue_isoc_tx()
3614 			(i ? ep_ring->cycle_state : !start_cycle);  in xhci_queue_isoc_tx()
3627 					ep_ring->cycle_state;  in xhci_queue_isoc_tx()
3639 				td->last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
3667 			queue_trb(xhci, ep_ring, more_trbs_coming,  in xhci_queue_isoc_tx()
3710 	urb_priv->td[0].last_trb = ep_ring->enqueue;  in xhci_queue_isoc_tx()
3712 	td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);  in xhci_queue_isoc_tx()
3715 	ep_ring->enqueue = urb_priv->td[0].first_trb;  in xhci_queue_isoc_tx()
3716 	ep_ring->enq_seg = urb_priv->td[0].start_seg;  in xhci_queue_isoc_tx()
3717 	ep_ring->cycle_state = start_cycle;  in xhci_queue_isoc_tx()
3718 	ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;  in xhci_queue_isoc_tx()
3734 	struct xhci_ring *ep_ring;  in xhci_queue_isoc_tx_prepare()  local
3744 	ep_ring = xdev->eps[ep_index].ring;  in xhci_queue_isoc_tx_prepare()
3755 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),  in xhci_queue_isoc_tx_prepare()
3767 	if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {  in xhci_queue_isoc_tx_prepare()
3800 	ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;  in xhci_queue_isoc_tx_prepare()