Lines Matching +full:generic +full:- +full:xhci
1 // SPDX-License-Identifier: GPL-2.0
3 * xHCI host controller driver
26 * until you reach a non-link TRB.
57 #include <linux/dma-mapping.h>
58 #include "xhci.h"
59 #include "xhci-trace.h"
61 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
74 if (!seg || !trb || trb < seg->trbs) in xhci_trb_virt_to_dma()
77 segment_offset = trb - seg->trbs; in xhci_trb_virt_to_dma()
80 return seg->dma + (segment_offset * sizeof(*trb)); in xhci_trb_virt_to_dma()
85 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); in trb_is_noop()
90 return TRB_TYPE_LINK_LE32(trb->link.control); in trb_is_link()
95 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; in last_trb_on_seg()
101 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); in last_trb_on_ring()
106 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; in link_trb_toggles_cycle()
111 struct urb_priv *urb_priv = td->urb->hcpriv; in last_td_in_urb()
113 return urb_priv->num_tds_done == urb_priv->num_tds; in last_td_in_urb()
118 struct urb_priv *urb_priv = urb->hcpriv; in inc_td_cnt()
120 urb_priv->num_tds_done++; in inc_td_cnt()
127 trb->link.control &= cpu_to_le32(~TRB_CHAIN); in trb_to_noop()
129 trb->generic.field[0] = 0; in trb_to_noop()
130 trb->generic.field[1] = 0; in trb_to_noop()
131 trb->generic.field[2] = 0; in trb_to_noop()
133 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); in trb_to_noop()
134 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); in trb_to_noop()
142 static void next_trb(struct xhci_hcd *xhci, in next_trb() argument
148 *seg = (*seg)->next; in next_trb()
149 *trb = ((*seg)->trbs); in next_trb()
158 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) in inc_deq() argument
163 if (ring->type == TYPE_EVENT) { in inc_deq()
164 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in inc_deq()
165 ring->dequeue++; in inc_deq()
168 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) in inc_deq()
169 ring->cycle_state ^= 1; in inc_deq()
170 ring->deq_seg = ring->deq_seg->next; in inc_deq()
171 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
176 if (!trb_is_link(ring->dequeue)) { in inc_deq()
177 if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) { in inc_deq()
178 xhci_warn(xhci, "Missing link TRB at end of segment\n"); in inc_deq()
180 ring->dequeue++; in inc_deq()
181 ring->num_trbs_free++; in inc_deq()
185 while (trb_is_link(ring->dequeue)) { in inc_deq()
186 ring->deq_seg = ring->deq_seg->next; in inc_deq()
187 ring->dequeue = ring->deq_seg->trbs; in inc_deq()
189 if (link_trb_count++ > ring->num_segs) { in inc_deq()
190 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in inc_deq()
211 * xHCI hardware can't handle the chain bit being cleared on a link TRB.
216 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, in inc_enq() argument
223 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; in inc_enq()
225 if (!trb_is_link(ring->enqueue)) in inc_enq()
226 ring->num_trbs_free--; in inc_enq()
228 if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) { in inc_enq()
229 xhci_err(xhci, "Tried to move enqueue past ring segment\n"); in inc_enq()
233 next = ++(ring->enqueue); in inc_enq()
252 if (!(ring->type == TYPE_ISOC && in inc_enq()
253 (xhci->quirks & XHCI_AMD_0x96_HOST)) && in inc_enq()
254 !xhci_link_trb_quirk(xhci)) { in inc_enq()
255 next->link.control &= cpu_to_le32(~TRB_CHAIN); in inc_enq()
256 next->link.control |= cpu_to_le32(chain); in inc_enq()
260 next->link.control ^= cpu_to_le32(TRB_CYCLE); in inc_enq()
264 ring->cycle_state ^= 1; in inc_enq()
266 ring->enq_seg = ring->enq_seg->next; in inc_enq()
267 ring->enqueue = ring->enq_seg->trbs; in inc_enq()
268 next = ring->enqueue; in inc_enq()
270 if (link_trb_count++ > ring->num_segs) { in inc_enq()
271 xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__); in inc_enq()
283 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, in room_on_ring() argument
288 if (ring->num_trbs_free < num_trbs) in room_on_ring()
291 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { in room_on_ring()
292 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; in room_on_ring()
293 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) in room_on_ring()
301 void xhci_ring_cmd_db(struct xhci_hcd *xhci) in xhci_ring_cmd_db() argument
303 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
306 xhci_dbg(xhci, "// Ding dong!\n"); in xhci_ring_cmd_db()
310 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
312 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
315 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay) in xhci_mod_cmd_timer() argument
317 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay); in xhci_mod_cmd_timer()
320 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) in xhci_next_queued_cmd() argument
322 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
327 * Turn all commands on command ring with status set to "aborted" to no-op trbs.
329 * This must be called with command ring stopped and xhci->lock held.
331 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, in xhci_handle_stopped_cmd_ring() argument
336 /* Turn all aborted commands in list to no-ops, then restart */ in xhci_handle_stopped_cmd_ring()
337 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
339 if (i_cmd->status != COMP_COMMAND_ABORTED) in xhci_handle_stopped_cmd_ring()
342 i_cmd->status = COMP_COMMAND_RING_STOPPED; in xhci_handle_stopped_cmd_ring()
344 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
345 i_cmd->command_trb); in xhci_handle_stopped_cmd_ring()
347 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP); in xhci_handle_stopped_cmd_ring()
351 * completion event is received for these no-op commands in xhci_handle_stopped_cmd_ring()
355 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
358 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
359 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
360 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
361 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in xhci_handle_stopped_cmd_ring()
362 xhci_ring_cmd_db(xhci); in xhci_handle_stopped_cmd_ring()
366 /* Must be called with xhci->lock held, releases and aquires lock back */
367 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) in xhci_abort_cmd_ring() argument
372 xhci_dbg(xhci, "Abort command ring\n"); in xhci_abort_cmd_ring()
374 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
383 temp_32 = readl(&xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
384 writel(temp_32 | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
386 /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the in xhci_abort_cmd_ring()
388 * seconds then driver handles it as if host died (-ENODEV). in xhci_abort_cmd_ring()
389 * In the future we should distinguish between -ENODEV and -ETIMEDOUT in xhci_abort_cmd_ring()
390 * and try to recover a -ETIMEDOUT with a host controller reset. in xhci_abort_cmd_ring()
392 ret = xhci_handshake(&xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
395 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); in xhci_abort_cmd_ring()
396 xhci_halt(xhci); in xhci_abort_cmd_ring()
397 xhci_hc_died(xhci); in xhci_abort_cmd_ring()
406 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
407 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
409 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
411 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); in xhci_abort_cmd_ring()
412 xhci_cleanup_command_queue(xhci); in xhci_abort_cmd_ring()
414 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); in xhci_abort_cmd_ring()
419 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, in xhci_ring_ep_doorbell() argument
424 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
425 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
426 unsigned int ep_state = ep->ep_state; in xhci_ring_ep_doorbell()
446 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in ring_doorbell_for_active_rings() argument
453 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
456 if (!(ep->ep_state & EP_HAS_STREAMS)) { in ring_doorbell_for_active_rings()
457 if (ep->ring && !(list_empty(&ep->ring->td_list))) in ring_doorbell_for_active_rings()
458 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); in ring_doorbell_for_active_rings()
462 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in ring_doorbell_for_active_rings()
464 struct xhci_stream_info *stream_info = ep->stream_info; in ring_doorbell_for_active_rings()
465 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) in ring_doorbell_for_active_rings()
466 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, in ring_doorbell_for_active_rings()
471 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in xhci_ring_doorbell_for_active_rings() argument
475 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_ring_doorbell_for_active_rings()
478 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci, in xhci_get_virt_ep() argument
483 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in xhci_get_virt_ep()
487 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index); in xhci_get_virt_ep()
490 if (!xhci->devs[slot_id]) { in xhci_get_virt_ep()
491 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id); in xhci_get_virt_ep()
495 return &xhci->devs[slot_id]->eps[ep_index]; in xhci_get_virt_ep()
498 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci, in xhci_virt_ep_to_ring() argument
503 if (!(ep->ep_state & EP_HAS_STREAMS)) in xhci_virt_ep_to_ring()
504 return ep->ring; in xhci_virt_ep_to_ring()
506 if (!ep->stream_info) in xhci_virt_ep_to_ring()
509 if (stream_id == 0 || stream_id >= ep->stream_info->num_streams) { in xhci_virt_ep_to_ring()
510 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n", in xhci_virt_ep_to_ring()
511 stream_id, ep->vdev->slot_id, ep->ep_index); in xhci_virt_ep_to_ring()
515 return ep->stream_info->stream_rings[stream_id]; in xhci_virt_ep_to_ring()
522 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, in xhci_triad_to_transfer_ring() argument
528 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_triad_to_transfer_ring()
532 return xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_triad_to_transfer_ring()
542 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, in xhci_get_hw_deq() argument
549 ep = &vdev->eps[ep_index]; in xhci_get_hw_deq()
551 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_get_hw_deq()
552 st_ctx = &ep->stream_info->stream_ctx_array[stream_id]; in xhci_get_hw_deq()
553 return le64_to_cpu(st_ctx->stream_ring); in xhci_get_hw_deq()
555 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
556 return le64_to_cpu(ep_ctx->deq); in xhci_get_hw_deq()
559 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, in xhci_move_dequeue_past_td() argument
563 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_move_dequeue_past_td()
564 struct xhci_virt_ep *ep = &dev->eps[ep_index]; in xhci_move_dequeue_past_td()
580 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, in xhci_move_dequeue_past_td()
583 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n", in xhci_move_dequeue_past_td()
585 return -ENODEV; in xhci_move_dequeue_past_td()
595 if (list_empty(&ep_ring->td_list)) { in xhci_move_dequeue_past_td()
596 new_seg = ep_ring->enq_seg; in xhci_move_dequeue_past_td()
597 new_deq = ep_ring->enqueue; in xhci_move_dequeue_past_td()
598 new_cycle = ep_ring->cycle_state; in xhci_move_dequeue_past_td()
599 xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue"); in xhci_move_dequeue_past_td()
602 xhci_warn(xhci, "Can't find new dequeue state, missing td\n"); in xhci_move_dequeue_past_td()
603 return -EINVAL; in xhci_move_dequeue_past_td()
607 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); in xhci_move_dequeue_past_td()
608 new_seg = ep_ring->deq_seg; in xhci_move_dequeue_past_td()
609 new_deq = ep_ring->dequeue; in xhci_move_dequeue_past_td()
612 * Quirk: xHC write-back of the DCS field in the hardware dequeue in xhci_move_dequeue_past_td()
613 * pointer is wrong - use the cycle state of the TRB pointed to by in xhci_move_dequeue_past_td()
616 if (xhci->quirks & XHCI_EP_CTX_BROKEN_DCS && in xhci_move_dequeue_past_td()
617 !(ep->ep_state & EP_HAS_STREAMS)) in xhci_move_dequeue_past_td()
618 halted_seg = trb_in_td(xhci, td->start_seg, in xhci_move_dequeue_past_td()
619 td->first_trb, td->last_trb, in xhci_move_dequeue_past_td()
622 index = ((dma_addr_t)(hw_dequeue & ~0xf) - halted_seg->dma) / in xhci_move_dequeue_past_td()
624 halted_trb = &halted_seg->trbs[index]; in xhci_move_dequeue_past_td()
625 new_cycle = halted_trb->generic.field[3] & 0x1; in xhci_move_dequeue_past_td()
626 xhci_dbg(xhci, "Endpoint DCS = %d TRB index = %d cycle = %d\n", in xhci_move_dequeue_past_td()
645 if (new_deq == td->last_trb) in xhci_move_dequeue_past_td()
652 next_trb(xhci, ep_ring, &new_seg, &new_deq); in xhci_move_dequeue_past_td()
655 if (new_deq == ep->ring->dequeue) { in xhci_move_dequeue_past_td()
656 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); in xhci_move_dequeue_past_td()
657 return -EINVAL; in xhci_move_dequeue_past_td()
667 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n"); in xhci_move_dequeue_past_td()
668 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq); in xhci_move_dequeue_past_td()
669 return -EINVAL; in xhci_move_dequeue_past_td()
672 if ((ep->ep_state & SET_DEQ_PENDING)) { in xhci_move_dequeue_past_td()
673 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n", in xhci_move_dequeue_past_td()
675 return -EBUSY; in xhci_move_dequeue_past_td()
679 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_move_dequeue_past_td()
681 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr); in xhci_move_dequeue_past_td()
682 return -ENOMEM; in xhci_move_dequeue_past_td()
687 ret = queue_command(xhci, cmd, in xhci_move_dequeue_past_td()
693 xhci_free_command(xhci, cmd); in xhci_move_dequeue_past_td()
696 ep->queued_deq_seg = new_seg; in xhci_move_dequeue_past_td()
697 ep->queued_deq_ptr = new_deq; in xhci_move_dequeue_past_td()
699 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_move_dequeue_past_td()
707 ep->ep_state |= SET_DEQ_PENDING; in xhci_move_dequeue_past_td()
708 xhci_ring_cmd_db(xhci); in xhci_move_dequeue_past_td()
716 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in td_to_noop() argument
719 struct xhci_segment *seg = td->start_seg; in td_to_noop()
720 union xhci_trb *trb = td->first_trb; in td_to_noop()
726 if (flip_cycle && trb != td->first_trb && trb != td->last_trb) in td_to_noop()
727 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); in td_to_noop()
729 if (trb == td->last_trb) in td_to_noop()
732 next_trb(xhci, ep_ring, &seg, &trb); in td_to_noop()
736 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci, in xhci_stop_watchdog_timer_in_irq() argument
739 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_stop_watchdog_timer_in_irq()
741 del_timer(&ep->stop_cmd_timer); in xhci_stop_watchdog_timer_in_irq()
745 * Must be called with xhci->lock held in interrupt context,
746 * releases and re-acquires xhci->lock
748 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, in xhci_giveback_urb_in_irq() argument
751 struct urb *urb = cur_td->urb; in xhci_giveback_urb_in_irq()
752 struct urb_priv *urb_priv = urb->hcpriv; in xhci_giveback_urb_in_irq()
753 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus); in xhci_giveback_urb_in_irq()
755 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) { in xhci_giveback_urb_in_irq()
756 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
757 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
758 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
768 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, in xhci_unmap_td_bounce_buffer() argument
771 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_unmap_td_bounce_buffer()
772 struct xhci_segment *seg = td->bounce_seg; in xhci_unmap_td_bounce_buffer()
773 struct urb *urb = td->urb; in xhci_unmap_td_bounce_buffer()
780 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
785 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, in xhci_unmap_td_bounce_buffer()
788 if (urb->num_sgs) { in xhci_unmap_td_bounce_buffer()
789 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
790 seg->bounce_len, seg->bounce_offs); in xhci_unmap_td_bounce_buffer()
791 if (len != seg->bounce_len) in xhci_unmap_td_bounce_buffer()
792 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n", in xhci_unmap_td_bounce_buffer()
793 len, seg->bounce_len); in xhci_unmap_td_bounce_buffer()
795 memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, in xhci_unmap_td_bounce_buffer()
796 seg->bounce_len); in xhci_unmap_td_bounce_buffer()
798 seg->bounce_len = 0; in xhci_unmap_td_bounce_buffer()
799 seg->bounce_offs = 0; in xhci_unmap_td_bounce_buffer()
802 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_td_cleanup() argument
808 urb = td->urb; in xhci_td_cleanup()
811 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); in xhci_td_cleanup()
815 * length, urb->actual_length will be a very big number (since it's in xhci_td_cleanup()
818 if (urb->actual_length > urb->transfer_buffer_length) { in xhci_td_cleanup()
819 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", in xhci_td_cleanup()
820 urb->transfer_buffer_length, urb->actual_length); in xhci_td_cleanup()
821 urb->actual_length = 0; in xhci_td_cleanup()
825 if (!list_empty(&td->td_list)) in xhci_td_cleanup()
826 list_del_init(&td->td_list); in xhci_td_cleanup()
828 if (!list_empty(&td->cancelled_td_list)) in xhci_td_cleanup()
829 list_del_init(&td->cancelled_td_list); in xhci_td_cleanup()
834 if ((urb->actual_length != urb->transfer_buffer_length && in xhci_td_cleanup()
835 (urb->transfer_flags & URB_SHORT_NOT_OK)) || in xhci_td_cleanup()
836 (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc))) in xhci_td_cleanup()
837 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", in xhci_td_cleanup()
838 urb, urb->actual_length, in xhci_td_cleanup()
839 urb->transfer_buffer_length, status); in xhci_td_cleanup()
842 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) in xhci_td_cleanup()
844 xhci_giveback_urb_in_irq(xhci, td, status); in xhci_td_cleanup()
857 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_giveback_invalidated_tds()
860 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_giveback_invalidated_tds()
862 if (td->cancel_status == TD_CLEARED) { in xhci_giveback_invalidated_tds()
863 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_giveback_invalidated_tds()
864 __func__, td->urb); in xhci_giveback_invalidated_tds()
865 xhci_td_cleanup(ep->xhci, td, ring, td->status); in xhci_giveback_invalidated_tds()
867 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_giveback_invalidated_tds()
868 __func__, td->urb, td->cancel_status); in xhci_giveback_invalidated_tds()
870 if (ep->xhci->xhc_state & XHCI_STATE_DYING) in xhci_giveback_invalidated_tds()
875 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, in xhci_reset_halted_ep() argument
881 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_reset_halted_ep()
883 ret = -ENOMEM; in xhci_reset_halted_ep()
887 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", in xhci_reset_halted_ep()
891 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); in xhci_reset_halted_ep()
894 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n", in xhci_reset_halted_ep()
899 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, in xhci_handle_halted_endpoint() argument
904 unsigned int slot_id = ep->vdev->slot_id; in xhci_handle_halted_endpoint()
911 if (ep->vdev->flags & VDEV_PORT_ERROR) in xhci_handle_halted_endpoint()
912 return -ENODEV; in xhci_handle_halted_endpoint()
916 ep->ep_state |= EP_HARD_CLEAR_TOGGLE; in xhci_handle_halted_endpoint()
917 if (td && list_empty(&td->cancelled_td_list)) { in xhci_handle_halted_endpoint()
918 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list); in xhci_handle_halted_endpoint()
919 td->cancel_status = TD_HALTED; in xhci_handle_halted_endpoint()
923 if (ep->ep_state & EP_HALTED) { in xhci_handle_halted_endpoint()
924 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n", in xhci_handle_halted_endpoint()
925 ep->ep_index); in xhci_handle_halted_endpoint()
929 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); in xhci_handle_halted_endpoint()
933 ep->ep_state |= EP_HALTED; in xhci_handle_halted_endpoint()
935 xhci_ring_cmd_db(xhci); in xhci_handle_halted_endpoint()
942 * We have the xHCI lock, so nothing can modify this list until we drop it.
943 * We're also in the event handler, so we can't get re-interrupted if another
951 struct xhci_hcd *xhci; in xhci_invalidate_cancelled_tds() local
957 unsigned int slot_id = ep->vdev->slot_id; in xhci_invalidate_cancelled_tds()
960 xhci = ep->xhci; in xhci_invalidate_cancelled_tds()
962 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
963 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_invalidate_cancelled_tds()
966 td->start_seg, td->first_trb), in xhci_invalidate_cancelled_tds()
967 td->urb->stream_id, td->urb); in xhci_invalidate_cancelled_tds()
968 list_del_init(&td->td_list); in xhci_invalidate_cancelled_tds()
969 ring = xhci_urb_to_transfer_ring(xhci, td->urb); in xhci_invalidate_cancelled_tds()
971 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n", in xhci_invalidate_cancelled_tds()
972 td->urb, td->urb->stream_id); in xhci_invalidate_cancelled_tds()
981 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, in xhci_invalidate_cancelled_tds()
982 td->urb->stream_id); in xhci_invalidate_cancelled_tds()
985 if (td->cancel_status == TD_HALTED || in xhci_invalidate_cancelled_tds()
986 trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) { in xhci_invalidate_cancelled_tds()
987 switch (td->cancel_status) { in xhci_invalidate_cancelled_tds()
988 case TD_CLEARED: /* TD is already no-op */ in xhci_invalidate_cancelled_tds()
993 td->cancel_status = TD_CLEARING_CACHE; in xhci_invalidate_cancelled_tds()
996 xhci_dbg(xhci, in xhci_invalidate_cancelled_tds()
998 td->urb->stream_id, td->urb, in xhci_invalidate_cancelled_tds()
999 cached_td->urb->stream_id, cached_td->urb); in xhci_invalidate_cancelled_tds()
1004 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1005 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1013 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, in xhci_invalidate_cancelled_tds()
1014 cached_td->urb->stream_id, in xhci_invalidate_cancelled_tds()
1017 /* Failed to move past cached td, just set cached TDs to no-op */ in xhci_invalidate_cancelled_tds()
1018 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { in xhci_invalidate_cancelled_tds()
1019 if (td->cancel_status != TD_CLEARING_CACHE) in xhci_invalidate_cancelled_tds()
1021 xhci_dbg(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", in xhci_invalidate_cancelled_tds()
1022 td->urb); in xhci_invalidate_cancelled_tds()
1023 td_to_noop(xhci, ring, td, false); in xhci_invalidate_cancelled_tds()
1024 td->cancel_status = TD_CLEARED; in xhci_invalidate_cancelled_tds()
1032 * Only call for non-running rings without streams.
1039 if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */ in find_halted_td()
1040 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); in find_halted_td()
1042 td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list); in find_halted_td()
1043 if (trb_in_td(ep->xhci, td->start_seg, td->first_trb, in find_halted_td()
1044 td->last_trb, hw_deq, false)) in find_halted_td()
1057 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
1060 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_stop_ep() argument
1071 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { in xhci_handle_cmd_stop_ep()
1072 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
1073 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n", in xhci_handle_cmd_stop_ep()
1078 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_stop_ep()
1079 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1083 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
1096 * Proper error code is unknown here, it would be -EPIPE if device side in xhci_handle_cmd_stop_ep()
1097 * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error) in xhci_handle_cmd_stop_ep()
1098 * We use -EPROTO, if device is stalled it should return a stall error on in xhci_handle_cmd_stop_ep()
1099 * next transfer, which then will return -EPIPE, and device side stall is in xhci_handle_cmd_stop_ep()
1104 xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n"); in xhci_handle_cmd_stop_ep()
1105 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_stop_ep()
1111 td->status = -EPROTO; in xhci_handle_cmd_stop_ep()
1114 err = xhci_handle_halted_endpoint(xhci, ep, 0, td, in xhci_handle_cmd_stop_ep()
1118 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1122 xhci_dbg(xhci, "Stop ep completion ctx error, ep is running\n"); in xhci_handle_cmd_stop_ep()
1124 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_handle_cmd_stop_ep()
1126 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1128 mod_timer(&ep->stop_cmd_timer, in xhci_handle_cmd_stop_ep()
1130 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0); in xhci_handle_cmd_stop_ep()
1131 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_stop_ep()
1140 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
1144 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
1147 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_kill_ring_urbs() argument
1152 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) { in xhci_kill_ring_urbs()
1153 list_del_init(&cur_td->td_list); in xhci_kill_ring_urbs()
1155 if (!list_empty(&cur_td->cancelled_td_list)) in xhci_kill_ring_urbs()
1156 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_ring_urbs()
1158 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); in xhci_kill_ring_urbs()
1160 inc_td_cnt(cur_td->urb); in xhci_kill_ring_urbs()
1162 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
1166 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, in xhci_kill_endpoint_urbs() argument
1174 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_kill_endpoint_urbs()
1175 if ((ep->ep_state & EP_HAS_STREAMS) || in xhci_kill_endpoint_urbs()
1176 (ep->ep_state & EP_GETTING_NO_STREAMS)) { in xhci_kill_endpoint_urbs()
1179 for (stream_id = 1; stream_id < ep->stream_info->num_streams; in xhci_kill_endpoint_urbs()
1181 ring = ep->stream_info->stream_rings[stream_id]; in xhci_kill_endpoint_urbs()
1185 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1188 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1191 ring = ep->ring; in xhci_kill_endpoint_urbs()
1194 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
1197 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
1200 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list, in xhci_kill_endpoint_urbs()
1202 list_del_init(&cur_td->cancelled_td_list); in xhci_kill_endpoint_urbs()
1203 inc_td_cnt(cur_td->urb); in xhci_kill_endpoint_urbs()
1206 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
1216 * Call with xhci->lock held.
1217 * lock is relased and re-acquired while giving back urb.
1219 void xhci_hc_died(struct xhci_hcd *xhci) in xhci_hc_died() argument
1223 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
1226 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); in xhci_hc_died()
1227 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
1229 xhci_cleanup_command_queue(xhci); in xhci_hc_died()
1232 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
1233 if (!xhci->devs[i]) in xhci_hc_died()
1236 xhci_kill_endpoint_urbs(xhci, i, j); in xhci_hc_died()
1240 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
1241 usb_hc_died(xhci_to_hcd(xhci)); in xhci_hc_died()
1248 * through xhci->state.
1264 struct xhci_hcd *xhci = ep->xhci; in xhci_stop_endpoint_command_watchdog() local
1269 spin_lock_irqsave(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1272 if (!(ep->ep_state & EP_STOP_CMD_PENDING) || in xhci_stop_endpoint_command_watchdog()
1273 timer_pending(&ep->stop_cmd_timer)) { in xhci_stop_endpoint_command_watchdog()
1274 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1275 xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit"); in xhci_stop_endpoint_command_watchdog()
1278 usbsts = readl(&xhci->op_regs->status); in xhci_stop_endpoint_command_watchdog()
1280 xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n"); in xhci_stop_endpoint_command_watchdog()
1281 xhci_warn(xhci, "USBSTS:%s\n", xhci_decode_usbsts(str, usbsts)); in xhci_stop_endpoint_command_watchdog()
1283 ep->ep_state &= ~EP_STOP_CMD_PENDING; in xhci_stop_endpoint_command_watchdog()
1285 xhci_halt(xhci); in xhci_stop_endpoint_command_watchdog()
1288 * handle a stop endpoint cmd timeout as if host died (-ENODEV). in xhci_stop_endpoint_command_watchdog()
1289 * In the future we could distinguish between -ENODEV and -ETIMEDOUT in xhci_stop_endpoint_command_watchdog()
1290 * and try to recover a -ETIMEDOUT with a host controller reset in xhci_stop_endpoint_command_watchdog()
1292 xhci_hc_died(xhci); in xhci_stop_endpoint_command_watchdog()
1294 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
1295 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_stop_endpoint_command_watchdog()
1296 "xHCI host controller is dead."); in xhci_stop_endpoint_command_watchdog()
1299 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci, in update_ring_for_set_deq_completion() argument
1308 num_trbs_free_temp = ep_ring->num_trbs_free; in update_ring_for_set_deq_completion()
1309 dequeue_temp = ep_ring->dequeue; in update_ring_for_set_deq_completion()
1311 /* If we get two back-to-back stalls, and the first stalled transfer in update_ring_for_set_deq_completion()
1315 * the segment into la-la-land. in update_ring_for_set_deq_completion()
1317 if (trb_is_link(ep_ring->dequeue)) { in update_ring_for_set_deq_completion()
1318 ep_ring->deq_seg = ep_ring->deq_seg->next; in update_ring_for_set_deq_completion()
1319 ep_ring->dequeue = ep_ring->deq_seg->trbs; in update_ring_for_set_deq_completion()
1322 while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) { in update_ring_for_set_deq_completion()
1324 ep_ring->num_trbs_free++; in update_ring_for_set_deq_completion()
1325 ep_ring->dequeue++; in update_ring_for_set_deq_completion()
1326 if (trb_is_link(ep_ring->dequeue)) { in update_ring_for_set_deq_completion()
1327 if (ep_ring->dequeue == in update_ring_for_set_deq_completion()
1328 dev->eps[ep_index].queued_deq_ptr) in update_ring_for_set_deq_completion()
1330 ep_ring->deq_seg = ep_ring->deq_seg->next; in update_ring_for_set_deq_completion()
1331 ep_ring->dequeue = ep_ring->deq_seg->trbs; in update_ring_for_set_deq_completion()
1333 if (ep_ring->dequeue == dequeue_temp) { in update_ring_for_set_deq_completion()
1340 xhci_dbg(xhci, "Unable to find new dequeue pointer\n"); in update_ring_for_set_deq_completion()
1341 ep_ring->num_trbs_free = num_trbs_free_temp; in update_ring_for_set_deq_completion()
1352 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_set_deq() argument
1363 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_set_deq()
1364 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2])); in xhci_handle_cmd_set_deq()
1365 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1369 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id); in xhci_handle_cmd_set_deq()
1371 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", in xhci_handle_cmd_set_deq()
1377 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1378 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in xhci_handle_cmd_set_deq()
1388 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); in xhci_handle_cmd_set_deq()
1391 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); in xhci_handle_cmd_set_deq()
1393 slot_state = le32_to_cpu(slot_ctx->dev_state); in xhci_handle_cmd_set_deq()
1395 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1400 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", in xhci_handle_cmd_set_deq()
1404 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", in xhci_handle_cmd_set_deq()
1417 if (ep->ep_state & EP_HAS_STREAMS) { in xhci_handle_cmd_set_deq()
1419 &ep->stream_info->stream_ctx_array[stream_id]; in xhci_handle_cmd_set_deq()
1420 deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK; in xhci_handle_cmd_set_deq()
1422 deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK; in xhci_handle_cmd_set_deq()
1424 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1426 if (xhci_trb_virt_to_dma(ep->queued_deq_seg, in xhci_handle_cmd_set_deq()
1427 ep->queued_deq_ptr) == deq) { in xhci_handle_cmd_set_deq()
1431 update_ring_for_set_deq_completion(xhci, ep->vdev, in xhci_handle_cmd_set_deq()
1434 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); in xhci_handle_cmd_set_deq()
1435 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", in xhci_handle_cmd_set_deq()
1436 ep->queued_deq_seg, ep->queued_deq_ptr); in xhci_handle_cmd_set_deq()
1440 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, in xhci_handle_cmd_set_deq()
1442 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); in xhci_handle_cmd_set_deq()
1443 if (td->cancel_status == TD_CLEARING_CACHE) { in xhci_handle_cmd_set_deq()
1444 td->cancel_status = TD_CLEARED; in xhci_handle_cmd_set_deq()
1445 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", in xhci_handle_cmd_set_deq()
1446 __func__, td->urb); in xhci_handle_cmd_set_deq()
1447 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); in xhci_handle_cmd_set_deq()
1449 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", in xhci_handle_cmd_set_deq()
1450 __func__, td->urb, td->cancel_status); in xhci_handle_cmd_set_deq()
1454 ep->ep_state &= ~SET_DEQ_PENDING; in xhci_handle_cmd_set_deq()
1455 ep->queued_deq_seg = NULL; in xhci_handle_cmd_set_deq()
1456 ep->queued_deq_ptr = NULL; in xhci_handle_cmd_set_deq()
1458 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1461 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_ep() argument
1468 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); in xhci_handle_cmd_reset_ep()
1469 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1473 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1479 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, in xhci_handle_cmd_reset_ep()
1485 if (xhci->quirks & XHCI_RESET_EP_QUIRK) in xhci_handle_cmd_reset_ep()
1486 xhci_dbg(xhci, "Note: Removed workaround to queue config ep for this hw"); in xhci_handle_cmd_reset_ep()
1488 ep->ep_state &= ~EP_HALTED; in xhci_handle_cmd_reset_ep()
1493 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) in xhci_handle_cmd_reset_ep()
1494 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_reset_ep()
1497 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_enable_slot() argument
1501 command->slot_id = slot_id; in xhci_handle_cmd_enable_slot()
1503 command->slot_id = 0; in xhci_handle_cmd_enable_slot()
1506 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_disable_slot() argument
1511 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1515 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1518 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1520 xhci_free_device_endpoint_resources(xhci, virt_dev, true); in xhci_handle_cmd_disable_slot()
1521 xhci_free_virt_device(xhci, slot_id); in xhci_handle_cmd_disable_slot()
1524 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_config_ep() argument
1539 * If the command was for a halted endpoint, the xHCI driver in xhci_handle_cmd_config_ep()
1542 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1545 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); in xhci_handle_cmd_config_ep()
1547 xhci_warn(xhci, "Could not get input context, bad type.\n"); in xhci_handle_cmd_config_ep()
1551 add_flags = le32_to_cpu(ctrl_ctx->add_flags); in xhci_handle_cmd_config_ep()
1552 drop_flags = le32_to_cpu(ctrl_ctx->drop_flags); in xhci_handle_cmd_config_ep()
1554 ep_index = xhci_last_valid_endpoint(add_flags) - 1; in xhci_handle_cmd_config_ep()
1556 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1565 if (xhci->quirks & XHCI_RESET_EP_QUIRK && in xhci_handle_cmd_config_ep()
1566 ep_index != (unsigned int) -1 && in xhci_handle_cmd_config_ep()
1567 add_flags - SLOT_FLAG == drop_flags) { in xhci_handle_cmd_config_ep()
1568 ep_state = virt_dev->eps[ep_index].ep_state; in xhci_handle_cmd_config_ep()
1571 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_config_ep()
1572 "Completed config ep cmd - " in xhci_handle_cmd_config_ep()
1576 virt_dev->eps[ep_index].ep_state &= ~EP_HALTED; in xhci_handle_cmd_config_ep()
1577 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_config_ep()
1583 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_addr_dev() argument
1588 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1591 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1595 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_reset_dev() argument
1600 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1602 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n", in xhci_handle_cmd_reset_dev()
1606 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1609 xhci_dbg(xhci, "Completed reset device command.\n"); in xhci_handle_cmd_reset_dev()
1612 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, in xhci_handle_cmd_nec_get_fw() argument
1615 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1616 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1619 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_nec_get_fw()
1621 NEC_FW_MAJOR(le32_to_cpu(event->status)), in xhci_handle_cmd_nec_get_fw()
1622 NEC_FW_MINOR(le32_to_cpu(event->status))); in xhci_handle_cmd_nec_get_fw()
1627 list_del(&cmd->cmd_list); in xhci_complete_del_and_free_cmd()
1629 if (cmd->completion) { in xhci_complete_del_and_free_cmd()
1630 cmd->status = status; in xhci_complete_del_and_free_cmd()
1631 complete(cmd->completion); in xhci_complete_del_and_free_cmd()
1637 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) in xhci_cleanup_command_queue() argument
1640 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1641 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1647 struct xhci_hcd *xhci; in xhci_handle_command_timeout() local
1651 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); in xhci_handle_command_timeout()
1653 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1659 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1660 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1664 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1667 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1669 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1673 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1676 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1677 xhci_dbg(xhci, "Command timeout\n"); in xhci_handle_command_timeout()
1678 xhci_abort_cmd_ring(xhci, flags); in xhci_handle_command_timeout()
1683 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1684 xhci_dbg(xhci, "host removed, ring start fail?\n"); in xhci_handle_command_timeout()
1685 xhci_cleanup_command_queue(xhci); in xhci_handle_command_timeout()
1691 xhci_dbg(xhci, "Command timeout on stopped ring\n"); in xhci_handle_command_timeout()
1692 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1695 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1699 static void handle_cmd_completion(struct xhci_hcd *xhci, in handle_cmd_completion() argument
1702 unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_cmd_completion()
1711 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); in handle_cmd_completion()
1715 cmd_dma = le64_to_cpu(event->cmd_trb); in handle_cmd_completion()
1716 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1718 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); in handle_cmd_completion()
1720 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1727 xhci_warn(xhci, in handle_cmd_completion()
1732 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1734 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1736 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); in handle_cmd_completion()
1740 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1744 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1745 xhci_err(xhci, in handle_cmd_completion()
1757 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1758 if (cmd->status == COMP_COMMAND_ABORTED) { in handle_cmd_completion()
1759 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1760 xhci->current_cmd = NULL; in handle_cmd_completion()
1765 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1768 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code); in handle_cmd_completion()
1771 xhci_handle_cmd_disable_slot(xhci, slot_id); in handle_cmd_completion()
1774 if (!cmd->completion) in handle_cmd_completion()
1775 xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code); in handle_cmd_completion()
1780 xhci_handle_cmd_addr_dev(xhci, slot_id); in handle_cmd_completion()
1784 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1785 if (!cmd->completion) in handle_cmd_completion()
1786 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, in handle_cmd_completion()
1791 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1792 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1795 /* Is this an aborted command turned to NO-OP? */ in handle_cmd_completion()
1796 if (cmd->status == COMP_COMMAND_RING_STOPPED) in handle_cmd_completion()
1801 le32_to_cpu(cmd_trb->generic.field[3]))); in handle_cmd_completion()
1802 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1806 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11) in handle_cmd_completion()
1809 le32_to_cpu(cmd_trb->generic.field[3])); in handle_cmd_completion()
1810 xhci_handle_cmd_reset_dev(xhci, slot_id); in handle_cmd_completion()
1813 xhci_handle_cmd_nec_get_fw(xhci, event); in handle_cmd_completion()
1817 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); in handle_cmd_completion()
1822 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1823 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1825 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in handle_cmd_completion()
1826 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1827 xhci->current_cmd = NULL; in handle_cmd_completion()
1833 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1836 static void handle_vendor_event(struct xhci_hcd *xhci, in handle_vendor_event() argument
1839 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); in handle_vendor_event()
1840 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1841 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1844 static void handle_device_notification(struct xhci_hcd *xhci, in handle_device_notification() argument
1850 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3])); in handle_device_notification()
1851 if (!xhci->devs[slot_id]) { in handle_device_notification()
1852 xhci_warn(xhci, "Device Notification event for " in handle_device_notification()
1857 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", in handle_device_notification()
1859 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1860 if (udev && udev->parent) in handle_device_notification()
1861 usb_wakeup_notification(udev->parent, udev->portnum); in handle_device_notification()
1865 * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI
1867 * As per ThunderX2errata-129 USB 2 device may come up as USB 1
1876 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) in xhci_cavium_reset_phy_quirk() argument
1878 struct usb_hcd *hcd = xhci_to_hcd(xhci); in xhci_cavium_reset_phy_quirk()
1884 writel(0x6F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1886 /* De-assert the PHY reset */ in xhci_cavium_reset_phy_quirk()
1887 writel(0x7F, hcd->regs + 0x1048); in xhci_cavium_reset_phy_quirk()
1889 pll_lock_check = readl(hcd->regs + 0x1070); in xhci_cavium_reset_phy_quirk()
1890 } while (!(pll_lock_check & 0x1) && --retry_count); in xhci_cavium_reset_phy_quirk()
1893 static void handle_port_status(struct xhci_hcd *xhci, in handle_port_status() argument
1907 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) in handle_port_status()
1908 xhci_warn(xhci, in handle_port_status()
1911 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); in handle_port_status()
1912 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1915 xhci_warn(xhci, "Port change event with invalid port ID %d\n", in handle_port_status()
1917 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1921 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
1922 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) { in handle_port_status()
1923 xhci_warn(xhci, "Port change event, no port for port ID %u\n", in handle_port_status()
1930 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { in handle_port_status()
1931 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); in handle_port_status()
1936 hcd = port->rhub->hcd; in handle_port_status()
1937 bus_state = &port->rhub->bus_state; in handle_port_status()
1938 hcd_portnum = port->hcd_portnum; in handle_port_status()
1939 portsc = readl(port->addr); in handle_port_status()
1941 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", in handle_port_status()
1942 hcd->self.busnum, hcd_portnum + 1, port_id, portsc); in handle_port_status()
1946 if (hcd->state == HC_STATE_SUSPENDED) { in handle_port_status()
1947 xhci_dbg(xhci, "resume root hub\n"); in handle_port_status()
1951 if (hcd->speed >= HCD_USB3 && in handle_port_status()
1953 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); in handle_port_status()
1954 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
1955 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR; in handle_port_status()
1959 xhci_dbg(xhci, "port resume event for port %d\n", port_id); in handle_port_status()
1961 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
1963 xhci_warn(xhci, "xHC is not running.\n"); in handle_port_status()
1968 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); in handle_port_status()
1973 bus_state->port_remote_wakeup |= 1 << hcd_portnum; in handle_port_status()
1974 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
1975 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
1976 xhci_set_link_state(xhci, port, XDEV_U0); in handle_port_status()
1982 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) { in handle_port_status()
1983 xhci_dbg(xhci, "resume HS port %d\n", port_id); in handle_port_status()
1984 bus_state->resume_done[hcd_portnum] = jiffies + in handle_port_status()
1986 set_bit(hcd_portnum, &bus_state->resuming_ports); in handle_port_status()
1989 * usb device auto-resume latency around ~40ms. in handle_port_status()
1991 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
1992 mod_timer(&hcd->rh_timer, in handle_port_status()
1993 bus_state->resume_done[hcd_portnum]); in handle_port_status()
1994 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); in handle_port_status()
2004 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); in handle_port_status()
2005 complete(&bus_state->u3exit_done[hcd_portnum]); in handle_port_status()
2008 * U3Exit state after a host-initiated resume. If it's a device in handle_port_status()
2013 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); in handle_port_status()
2014 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
2015 xhci_ring_device(xhci, slot_id); in handle_port_status()
2016 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { in handle_port_status()
2017 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2018 usb_wakeup_notification(hcd->self.root_hub, in handle_port_status()
2026 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or in handle_port_status()
2030 if (!DEV_SUPERSPEED_ANY(portsc) && hcd->speed < HCD_USB3 && in handle_port_status()
2032 &bus_state->rexit_ports)) { in handle_port_status()
2033 complete(&bus_state->rexit_done[hcd_portnum]); in handle_port_status()
2038 if (hcd->speed < HCD_USB3) { in handle_port_status()
2039 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
2040 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && in handle_port_status()
2042 xhci_cavium_reset_phy_quirk(xhci); in handle_port_status()
2047 inc_deq(xhci, xhci->event_ring); in handle_port_status()
2057 * xHCI port-status-change events occur when the "or" of all the in handle_port_status()
2058 * status-change bits in the portsc register changes from 0 to 1. in handle_port_status()
2063 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", in handle_port_status()
2064 __func__, hcd->self.busnum); in handle_port_status()
2065 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); in handle_port_status()
2066 spin_unlock(&xhci->lock); in handle_port_status()
2069 spin_lock(&xhci->lock); in handle_port_status()
2078 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci, in trb_in_td() argument
2098 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]); in trb_in_td()
2103 xhci_warn(xhci, in trb_in_td()
2104 …"Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx… in trb_in_td()
2108 (unsigned long long)cur_seg->dma, in trb_in_td()
2122 (suspect_dma >= cur_seg->dma && in trb_in_td()
2132 cur_seg = cur_seg->next; in trb_in_td()
2133 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); in trb_in_td()
2139 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_clear_hub_tt_buffer() argument
2143 * As part of low/full-speed endpoint-halt processing in xhci_clear_hub_tt_buffer()
2146 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) && in xhci_clear_hub_tt_buffer()
2147 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && in xhci_clear_hub_tt_buffer()
2148 !(ep->ep_state & EP_CLEARING_TT)) { in xhci_clear_hub_tt_buffer()
2149 ep->ep_state |= EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2150 td->urb->ep->hcpriv = td->urb->dev; in xhci_clear_hub_tt_buffer()
2151 if (usb_hub_clear_tt_buffer(td->urb)) in xhci_clear_hub_tt_buffer()
2152 ep->ep_state &= ~EP_CLEARING_TT; in xhci_clear_hub_tt_buffer()
2157 * cleanup the halt for a non-default control endpoint if we indicate a stall.
2162 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci, in xhci_requires_manual_halt_cleanup() argument
2182 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) in xhci_is_vendor_info_code() argument
2186 * treat as not-an-error. in xhci_is_vendor_info_code()
2188 xhci_dbg(xhci, "Vendor defined info completion code %u\n", in xhci_is_vendor_info_code()
2190 xhci_dbg(xhci, "Treating code as success.\n"); in xhci_is_vendor_info_code()
2196 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in finish_td() argument
2202 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in finish_td()
2234 if ((ep->ep_state & EP_HALTED) && in finish_td()
2235 !list_empty(&td->cancelled_td_list)) { in finish_td()
2236 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n", in finish_td()
2238 td->start_seg, td->first_trb)); in finish_td()
2245 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2246 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in finish_td()
2251 * xhci internal endpoint state will go to a "halt" state for in finish_td()
2260 if (ep->ep_index != 0) in finish_td()
2261 xhci_clear_hub_tt_buffer(xhci, td, ep); in finish_td()
2263 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in finish_td()
2272 ep_ring->dequeue = td->last_trb; in finish_td()
2273 ep_ring->deq_seg = td->last_trb_seg; in finish_td()
2274 ep_ring->num_trbs_free += td->num_trbs - 1; in finish_td()
2275 inc_deq(xhci, ep_ring); in finish_td()
2277 return xhci_td_cleanup(xhci, td, ep_ring, td->status); in finish_td()
2281 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, in sum_trb_lengths() argument
2285 union xhci_trb *trb = ring->dequeue; in sum_trb_lengths()
2286 struct xhci_segment *seg = ring->deq_seg; in sum_trb_lengths()
2288 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) { in sum_trb_lengths()
2290 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); in sum_trb_lengths()
2298 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_ctrl_td() argument
2307 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3])); in process_ctrl_td()
2308 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); in process_ctrl_td()
2309 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2310 requested = td->urb->transfer_buffer_length; in process_ctrl_td()
2311 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_ctrl_td()
2316 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", in process_ctrl_td()
2318 td->status = -ESHUTDOWN; in process_ctrl_td()
2321 td->status = 0; in process_ctrl_td()
2324 td->status = 0; in process_ctrl_td()
2328 td->urb->actual_length = remaining; in process_ctrl_td()
2330 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); in process_ctrl_td()
2335 td->urb->actual_length = 0; in process_ctrl_td()
2339 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2342 td->urb->actual_length = requested; in process_ctrl_td()
2345 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", in process_ctrl_td()
2352 if (!xhci_requires_manual_halt_cleanup(xhci, in process_ctrl_td()
2355 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", in process_ctrl_td()
2356 trb_comp_code, ep->ep_index); in process_ctrl_td()
2361 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2362 else if (!td->urb_length_set) in process_ctrl_td()
2363 td->urb->actual_length = 0; in process_ctrl_td()
2377 td->urb_length_set = true; in process_ctrl_td()
2378 td->urb->actual_length = requested - remaining; in process_ctrl_td()
2379 xhci_dbg(xhci, "Waiting for status stage event\n"); in process_ctrl_td()
2384 if (!td->urb_length_set) in process_ctrl_td()
2385 td->urb->actual_length = requested; in process_ctrl_td()
2388 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_ctrl_td()
2394 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_isoc_td() argument
2406 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2407 urb_priv = td->urb->hcpriv; in process_isoc_td()
2408 idx = urb_priv->num_tds_done; in process_isoc_td()
2409 frame = &td->urb->iso_frame_desc[idx]; in process_isoc_td()
2410 requested = frame->length; in process_isoc_td()
2411 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_isoc_td()
2412 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_isoc_td()
2413 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ? in process_isoc_td()
2414 -EREMOTEIO : 0; in process_isoc_td()
2420 frame->status = short_framestatus; in process_isoc_td()
2421 if (xhci->quirks & XHCI_TRUST_TX_LENGTH) in process_isoc_td()
2425 frame->status = 0; in process_isoc_td()
2428 frame->status = short_framestatus; in process_isoc_td()
2432 frame->status = -ECOMM; in process_isoc_td()
2436 frame->status = -EOVERFLOW; in process_isoc_td()
2440 frame->status = -EPROTO; in process_isoc_td()
2443 frame->status = -EPROTO; in process_isoc_td()
2444 if (ep_trb != td->last_trb) in process_isoc_td()
2452 frame->status = short_framestatus; in process_isoc_td()
2461 frame->status = -1; in process_isoc_td()
2466 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) + in process_isoc_td()
2467 ep_trb_len - remaining; in process_isoc_td()
2469 frame->actual_length = requested; in process_isoc_td()
2471 td->urb->actual_length += frame->actual_length; in process_isoc_td()
2473 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_isoc_td()
2476 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in skip_isoc_td() argument
2483 urb_priv = td->urb->hcpriv; in skip_isoc_td()
2484 idx = urb_priv->num_tds_done; in skip_isoc_td()
2485 frame = &td->urb->iso_frame_desc[idx]; in skip_isoc_td()
2488 frame->status = -EXDEV; in skip_isoc_td()
2491 frame->actual_length = 0; in skip_isoc_td()
2494 ep->ring->dequeue = td->last_trb; in skip_isoc_td()
2495 ep->ring->deq_seg = td->last_trb_seg; in skip_isoc_td()
2496 ep->ring->num_trbs_free += td->num_trbs - 1; in skip_isoc_td()
2497 inc_deq(xhci, ep->ring); in skip_isoc_td()
2499 return xhci_td_cleanup(xhci, td, ep->ring, status); in skip_isoc_td()
2505 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, in process_bulk_intr_td() argument
2513 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); in process_bulk_intr_td()
2514 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2515 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); in process_bulk_intr_td()
2516 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); in process_bulk_intr_td()
2517 requested = td->urb->transfer_buffer_length; in process_bulk_intr_td()
2521 ep_ring->err_count = 0; in process_bulk_intr_td()
2523 if (ep_trb != td->last_trb || remaining) { in process_bulk_intr_td()
2524 xhci_warn(xhci, "WARN Successful completion on short TX\n"); in process_bulk_intr_td()
2525 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2526 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2529 td->status = 0; in process_bulk_intr_td()
2532 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2533 td->urb->ep->desc.bEndpointAddress, in process_bulk_intr_td()
2535 td->status = 0; in process_bulk_intr_td()
2538 td->urb->actual_length = remaining; in process_bulk_intr_td()
2546 if (xhci->quirks & XHCI_NO_SOFT_RETRY || in process_bulk_intr_td()
2547 (ep_ring->err_count++ > MAX_SOFT_RETRY) || in process_bulk_intr_td()
2548 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) in process_bulk_intr_td()
2551 td->status = 0; in process_bulk_intr_td()
2553 xhci_handle_halted_endpoint(xhci, ep, ep_ring->stream_id, td, in process_bulk_intr_td()
2561 if (ep_trb == td->last_trb) in process_bulk_intr_td()
2562 td->urb->actual_length = requested - remaining; in process_bulk_intr_td()
2564 td->urb->actual_length = in process_bulk_intr_td()
2565 sum_trb_lengths(xhci, ep_ring, ep_trb) + in process_bulk_intr_td()
2566 ep_trb_len - remaining; in process_bulk_intr_td()
2569 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", in process_bulk_intr_td()
2571 td->urb->actual_length = 0; in process_bulk_intr_td()
2574 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); in process_bulk_intr_td()
2582 static int handle_tx_event(struct xhci_hcd *xhci, in handle_tx_event() argument
2593 int status = -EINPROGRESS; in handle_tx_event()
2600 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); in handle_tx_event()
2601 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; in handle_tx_event()
2602 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); in handle_tx_event()
2603 ep_trb_dma = le64_to_cpu(event->buffer); in handle_tx_event()
2605 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); in handle_tx_event()
2607 xhci_err(xhci, "ERROR Invalid Transfer event\n"); in handle_tx_event()
2612 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); in handle_tx_event()
2615 xhci_err(xhci, in handle_tx_event()
2621 /* Some transfer events don't always point to a trb, see xhci 4.17.4 */ in handle_tx_event()
2628 xhci_handle_halted_endpoint(xhci, ep, 0, NULL, in handle_tx_event()
2636 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n", in handle_tx_event()
2642 /* Count current td numbers if ep->skip is set */ in handle_tx_event()
2643 if (ep->skip) { in handle_tx_event()
2644 list_for_each(tmp, &ep_ring->td_list) in handle_tx_event()
2654 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0) in handle_tx_event()
2656 if (xhci->quirks & XHCI_TRUST_TX_LENGTH || in handle_tx_event()
2657 ep_ring->last_td_was_short) in handle_tx_event()
2660 xhci_warn_ratelimited(xhci, in handle_tx_event()
2668 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", in handle_tx_event()
2672 xhci_dbg(xhci, in handle_tx_event()
2673 "Stopped on No-op or Link TRB for slot %u ep %u\n", in handle_tx_event()
2677 xhci_dbg(xhci, in handle_tx_event()
2683 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, in handle_tx_event()
2685 status = -EPIPE; in handle_tx_event()
2688 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n", in handle_tx_event()
2690 status = -EPROTO; in handle_tx_event()
2693 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", in handle_tx_event()
2695 status = -EPROTO; in handle_tx_event()
2698 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", in handle_tx_event()
2700 status = -EOVERFLOW; in handle_tx_event()
2704 xhci_warn(xhci, in handle_tx_event()
2707 status = -EILSEQ; in handle_tx_event()
2711 xhci_warn(xhci, in handle_tx_event()
2714 status = -ENOSR; in handle_tx_event()
2717 xhci_warn(xhci, in handle_tx_event()
2722 xhci_warn(xhci, in handle_tx_event()
2732 xhci_dbg(xhci, "underrun event on endpoint\n"); in handle_tx_event()
2733 if (!list_empty(&ep_ring->td_list)) in handle_tx_event()
2734 xhci_dbg(xhci, "Underrun Event for slot %d ep %d " in handle_tx_event()
2736 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2740 xhci_dbg(xhci, "overrun event on endpoint\n"); in handle_tx_event()
2741 if (!list_empty(&ep_ring->td_list)) in handle_tx_event()
2742 xhci_dbg(xhci, "Overrun Event for slot %d ep %d " in handle_tx_event()
2744 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2754 ep->skip = true; in handle_tx_event()
2755 xhci_dbg(xhci, in handle_tx_event()
2760 ep->skip = true; in handle_tx_event()
2761 xhci_dbg(xhci, in handle_tx_event()
2768 xhci_warn(xhci, in handle_tx_event()
2771 status = -EPROTO; in handle_tx_event()
2774 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { in handle_tx_event()
2778 xhci_warn(xhci, in handle_tx_event()
2788 if (list_empty(&ep_ring->td_list)) { in handle_tx_event()
2799 ep_ring->last_td_was_short)) { in handle_tx_event()
2800 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n", in handle_tx_event()
2801 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), in handle_tx_event()
2804 if (ep->skip) { in handle_tx_event()
2805 ep->skip = false; in handle_tx_event()
2806 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2810 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in handle_tx_event()
2812 xhci_handle_halted_endpoint(xhci, ep, in handle_tx_event()
2813 ep_ring->stream_id, in handle_tx_event()
2820 /* We've skipped all the TDs on the ep ring when ep->skip set */ in handle_tx_event()
2821 if (ep->skip && td_num == 0) { in handle_tx_event()
2822 ep->skip = false; in handle_tx_event()
2823 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2828 td = list_first_entry(&ep_ring->td_list, struct xhci_td, in handle_tx_event()
2830 if (ep->skip) in handle_tx_event()
2831 td_num--; in handle_tx_event()
2834 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue, in handle_tx_event()
2835 td->last_trb, ep_trb_dma, false); in handle_tx_event()
2839 * is not in the current TD pointed by ep_ring->dequeue because in handle_tx_event()
2851 if (!ep->skip || in handle_tx_event()
2852 !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) { in handle_tx_event()
2857 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && in handle_tx_event()
2858 ep_ring->last_td_was_short) { in handle_tx_event()
2859 ep_ring->last_td_was_short = false; in handle_tx_event()
2863 xhci_err(xhci, in handle_tx_event()
2868 trb_in_td(xhci, ep_ring->deq_seg, in handle_tx_event()
2869 ep_ring->dequeue, td->last_trb, in handle_tx_event()
2871 return -ESHUTDOWN; in handle_tx_event()
2874 skip_isoc_td(xhci, td, ep, status); in handle_tx_event()
2878 ep_ring->last_td_was_short = true; in handle_tx_event()
2880 ep_ring->last_td_was_short = false; in handle_tx_event()
2882 if (ep->skip) { in handle_tx_event()
2883 xhci_dbg(xhci, in handle_tx_event()
2886 ep->skip = false; in handle_tx_event()
2889 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) / in handle_tx_event()
2896 * No-op TRB could trigger interrupts in a case where in handle_tx_event()
2905 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in handle_tx_event()
2907 xhci_handle_halted_endpoint(xhci, ep, in handle_tx_event()
2908 ep_ring->stream_id, in handle_tx_event()
2913 td->status = status; in handle_tx_event()
2916 if (usb_endpoint_xfer_control(&td->urb->ep->desc)) in handle_tx_event()
2917 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2918 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) in handle_tx_event()
2919 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2921 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); in handle_tx_event()
2923 handling_skipped_tds = ep->skip && in handle_tx_event()
2932 inc_deq(xhci, xhci->event_ring); in handle_tx_event()
2935 * If ep->skip is set, it means there are missed tds on the in handle_tx_event()
2945 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", in handle_tx_event()
2947 xhci->event_ring->deq_seg, in handle_tx_event()
2948 xhci->event_ring->dequeue), in handle_tx_event()
2949 lower_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2950 upper_32_bits(le64_to_cpu(event->buffer)), in handle_tx_event()
2951 le32_to_cpu(event->transfer_len), in handle_tx_event()
2952 le32_to_cpu(event->flags)); in handle_tx_event()
2953 return -ENODEV; in handle_tx_event()
2957 * This function handles all OS-owned events on the event ring. It may drop
2958 * xhci->lock between event processing (e.g. to pass up port status changes).
2962 static int xhci_handle_event(struct xhci_hcd *xhci) in xhci_handle_event() argument
2970 if (!xhci->event_ring || !xhci->event_ring->dequeue) { in xhci_handle_event()
2971 xhci_err(xhci, "ERROR event ring not ready\n"); in xhci_handle_event()
2972 return -ENOMEM; in xhci_handle_event()
2975 event = xhci->event_ring->dequeue; in xhci_handle_event()
2977 if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) != in xhci_handle_event()
2978 xhci->event_ring->cycle_state) in xhci_handle_event()
2981 trace_xhci_handle_event(xhci->event_ring, &event->generic); in xhci_handle_event()
2988 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags)); in xhci_handle_event()
2993 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event()
2996 handle_port_status(xhci, event); in xhci_handle_event()
3000 ret = handle_tx_event(xhci, &event->trans_event); in xhci_handle_event()
3005 handle_device_notification(xhci, event); in xhci_handle_event()
3009 handle_vendor_event(xhci, event, trb_type); in xhci_handle_event()
3011 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type); in xhci_handle_event()
3013 /* Any of the above functions may drop and re-acquire the lock, so check in xhci_handle_event()
3014 * to make sure a watchdog timer didn't mark the host as non-responsive. in xhci_handle_event()
3016 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event()
3017 xhci_dbg(xhci, "xHCI host dying, returning from " in xhci_handle_event()
3024 inc_deq(xhci, xhci->event_ring); in xhci_handle_event()
3034 * - When all events have finished
3035 * - To avoid "Event Ring Full Error" condition
3037 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci, in xhci_update_erst_dequeue() argument
3043 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3045 if (event_ring_deq != xhci->event_ring->dequeue) { in xhci_update_erst_dequeue()
3046 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_update_erst_dequeue()
3047 xhci->event_ring->dequeue); in xhci_update_erst_dequeue()
3049 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n"); in xhci_update_erst_dequeue()
3065 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue); in xhci_update_erst_dequeue()
3069 * xHCI spec says we can get an interrupt, and if the HC has an error condition,
3075 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_irq() local
3082 spin_lock(&xhci->lock); in xhci_irq()
3084 status = readl(&xhci->op_regs->status); in xhci_irq()
3086 xhci_hc_died(xhci); in xhci_irq()
3095 xhci_warn(xhci, "WARNING: Host System Error\n"); in xhci_irq()
3096 xhci_halt(xhci); in xhci_irq()
3103 * so we can receive interrupts from other MSI-X interrupters. in xhci_irq()
3107 writel(status, &xhci->op_regs->status); in xhci_irq()
3109 if (!hcd->msi_enabled) { in xhci_irq()
3111 irq_pending = readl(&xhci->ir_set->irq_pending); in xhci_irq()
3113 writel(irq_pending, &xhci->ir_set->irq_pending); in xhci_irq()
3116 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_irq()
3117 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_irq()
3118 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. " in xhci_irq()
3123 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_irq()
3124 xhci_write_64(xhci, temp_64 | ERST_EHB, in xhci_irq()
3125 &xhci->ir_set->erst_dequeue); in xhci_irq()
3130 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
3134 while (xhci_handle_event(xhci) > 0) { in xhci_irq()
3137 xhci_update_erst_dequeue(xhci, event_ring_deq); in xhci_irq()
3139 /* ring is half-full, force isoc trbs to interrupt more often */ in xhci_irq()
3140 if (xhci->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN) in xhci_irq()
3141 xhci->isoc_bei_interval = xhci->isoc_bei_interval / 2; in xhci_irq()
3146 xhci_update_erst_dequeue(xhci, event_ring_deq); in xhci_irq()
3150 spin_unlock(&xhci->lock); in xhci_irq()
3163 * Generic function for queueing a TRB on a ring.
3169 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, in queue_trb() argument
3175 trb = &ring->enqueue->generic; in queue_trb()
3176 trb->field[0] = cpu_to_le32(field1); in queue_trb()
3177 trb->field[1] = cpu_to_le32(field2); in queue_trb()
3178 trb->field[2] = cpu_to_le32(field3); in queue_trb()
3181 trb->field[3] = cpu_to_le32(field4); in queue_trb()
3185 inc_enq(xhci, ring, more_trbs_coming); in queue_trb()
3192 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in prepare_ring() argument
3205 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); in prepare_ring()
3206 return -ENOENT; in prepare_ring()
3208 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); in prepare_ring()
3210 /* XXX not sure if this should be -ENOENT or not */ in prepare_ring()
3211 return -EINVAL; in prepare_ring()
3213 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); in prepare_ring()
3219 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); in prepare_ring()
3224 return -EINVAL; in prepare_ring()
3228 if (room_on_ring(xhci, ep_ring, num_trbs)) in prepare_ring()
3231 if (ep_ring == xhci->cmd_ring) { in prepare_ring()
3232 xhci_err(xhci, "Do not support expand command ring\n"); in prepare_ring()
3233 return -ENOMEM; in prepare_ring()
3236 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in prepare_ring()
3238 num_trbs_needed = num_trbs - ep_ring->num_trbs_free; in prepare_ring()
3239 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed, in prepare_ring()
3241 xhci_err(xhci, "Ring expansion failed\n"); in prepare_ring()
3242 return -ENOMEM; in prepare_ring()
3246 while (trb_is_link(ep_ring->enqueue)) { in prepare_ring()
3250 if (!xhci_link_trb_quirk(xhci) && in prepare_ring()
3251 !(ep_ring->type == TYPE_ISOC && in prepare_ring()
3252 (xhci->quirks & XHCI_AMD_0x96_HOST))) in prepare_ring()
3253 ep_ring->enqueue->link.control &= in prepare_ring()
3256 ep_ring->enqueue->link.control |= in prepare_ring()
3260 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); in prepare_ring()
3263 if (link_trb_toggles_cycle(ep_ring->enqueue)) in prepare_ring()
3264 ep_ring->cycle_state ^= 1; in prepare_ring()
3266 ep_ring->enq_seg = ep_ring->enq_seg->next; in prepare_ring()
3267 ep_ring->enqueue = ep_ring->enq_seg->trbs; in prepare_ring()
3270 if (link_trb_count++ > ep_ring->num_segs) { in prepare_ring()
3271 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); in prepare_ring()
3272 return -EINVAL; in prepare_ring()
3276 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) { in prepare_ring()
3277 xhci_warn(xhci, "Missing link TRB at end of ring segment\n"); in prepare_ring()
3278 return -EINVAL; in prepare_ring()
3284 static int prepare_transfer(struct xhci_hcd *xhci, in prepare_transfer() argument
3297 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
3299 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, in prepare_transfer()
3302 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", in prepare_transfer()
3304 return -EINVAL; in prepare_transfer()
3307 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in prepare_transfer()
3312 urb_priv = urb->hcpriv; in prepare_transfer()
3313 td = &urb_priv->td[td_index]; in prepare_transfer()
3315 INIT_LIST_HEAD(&td->td_list); in prepare_transfer()
3316 INIT_LIST_HEAD(&td->cancelled_td_list); in prepare_transfer()
3319 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb); in prepare_transfer()
3324 td->urb = urb; in prepare_transfer()
3326 list_add_tail(&td->td_list, &ep_ring->td_list); in prepare_transfer()
3327 td->start_seg = ep_ring->enq_seg; in prepare_transfer()
3328 td->first_trb = ep_ring->enqueue; in prepare_transfer()
3337 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), in count_trbs()
3347 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length); in count_trbs_needed()
3355 full_len = urb->transfer_buffer_length; in count_sg_trbs_needed()
3357 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) { in count_sg_trbs_needed()
3361 full_len -= len; in count_sg_trbs_needed()
3373 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); in count_isoc_trbs_needed()
3374 len = urb->iso_frame_desc[i].length; in count_isoc_trbs_needed()
3381 if (unlikely(running_total != urb->transfer_buffer_length)) in check_trb_math()
3382 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " in check_trb_math()
3385 urb->ep->desc.bEndpointAddress, in check_trb_math()
3387 urb->transfer_buffer_length, in check_trb_math()
3388 urb->transfer_buffer_length); in check_trb_math()
3391 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, in giveback_first_trb() argument
3401 start_trb->field[3] |= cpu_to_le32(start_cycle); in giveback_first_trb()
3403 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); in giveback_first_trb()
3404 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); in giveback_first_trb()
3407 static void check_interval(struct xhci_hcd *xhci, struct urb *urb, in check_interval() argument
3413 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info)); in check_interval()
3414 ep_interval = urb->interval; in check_interval()
3417 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3418 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3425 dev_dbg_ratelimited(&urb->dev->dev, in check_interval()
3426 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n", in check_interval()
3429 urb->interval = xhci_interval; in check_interval()
3431 if (urb->dev->speed == USB_SPEED_LOW || in check_interval()
3432 urb->dev->speed == USB_SPEED_FULL) in check_interval()
3433 urb->interval /= 8; in check_interval()
3438 * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt
3443 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_intr_tx() argument
3448 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3449 check_interval(xhci, urb, ep_ctx); in xhci_queue_intr_tx()
3451 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_intr_tx()
3455 * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3464 * TD size = total_packet_count - packets_transferred
3466 * For xHCI 0.96 and older, TD size field should be the remaining bytes
3474 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, in xhci_td_remainder() argument
3480 /* MTK xHCI 0.96 contains some features from 1.0 */ in xhci_td_remainder()
3481 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3482 return ((td_total_len - transferred) >> 10); in xhci_td_remainder()
3484 /* One TRB with a zero-length data packet. */ in xhci_td_remainder()
3489 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */ in xhci_td_remainder()
3490 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3493 maxp = usb_endpoint_maxp(&urb->ep->desc); in xhci_td_remainder()
3497 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); in xhci_td_remainder()
3501 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, in xhci_align_td() argument
3504 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_align_td()
3510 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_align_td()
3517 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", in xhci_align_td()
3522 *trb_buff_len -= unalign; in xhci_align_td()
3523 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3532 new_buff_len = max_pkt - (enqd_len % max_pkt); in xhci_align_td()
3534 if (new_buff_len > (urb->transfer_buffer_length - enqd_len)) in xhci_align_td()
3535 new_buff_len = (urb->transfer_buffer_length - enqd_len); in xhci_align_td()
3539 if (urb->num_sgs) { in xhci_align_td()
3540 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs, in xhci_align_td()
3541 seg->bounce_buf, new_buff_len, enqd_len); in xhci_align_td()
3543 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n", in xhci_align_td()
3546 memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len); in xhci_align_td()
3549 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3552 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, in xhci_align_td()
3556 if (dma_mapping_error(dev, seg->bounce_dma)) { in xhci_align_td()
3558 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); in xhci_align_td()
3562 seg->bounce_len = new_buff_len; in xhci_align_td()
3563 seg->bounce_offs = enqd_len; in xhci_align_td()
3565 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3570 /* This is very similar to what ehci-q.c qtd_fill() does */
3571 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_bulk_tx() argument
3589 ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_bulk_tx()
3591 return -EINVAL; in xhci_queue_bulk_tx()
3593 full_len = urb->transfer_buffer_length; in xhci_queue_bulk_tx()
3595 if (urb->num_sgs && !(urb->transfer_flags & URB_DMA_MAP_SINGLE)) { in xhci_queue_bulk_tx()
3596 num_sgs = urb->num_mapped_sgs; in xhci_queue_bulk_tx()
3597 sg = urb->sg; in xhci_queue_bulk_tx()
3603 addr = (u64) urb->transfer_dma; in xhci_queue_bulk_tx()
3606 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3607 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3612 urb_priv = urb->hcpriv; in xhci_queue_bulk_tx()
3614 /* Deal with URB_ZERO_PACKET - need one more td/trb */ in xhci_queue_bulk_tx()
3615 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1) in xhci_queue_bulk_tx()
3618 td = &urb_priv->td[0]; in xhci_queue_bulk_tx()
3625 start_trb = &ring->enqueue->generic; in xhci_queue_bulk_tx()
3626 start_cycle = ring->cycle_state; in xhci_queue_bulk_tx()
3629 /* Queue the TRBs, even if they are zero-length */ in xhci_queue_bulk_tx()
3639 trb_buff_len = full_len - enqd_len; in xhci_queue_bulk_tx()
3647 field |= ring->cycle_state; in xhci_queue_bulk_tx()
3654 if (trb_is_link(ring->enqueue + 1)) { in xhci_queue_bulk_tx()
3655 if (xhci_align_td(xhci, urb, enqd_len, in xhci_queue_bulk_tx()
3657 ring->enq_seg)) { in xhci_queue_bulk_tx()
3658 send_addr = ring->enq_seg->bounce_dma; in xhci_queue_bulk_tx()
3660 td->bounce_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3668 td->last_trb = ring->enqueue; in xhci_queue_bulk_tx()
3669 td->last_trb_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3671 memcpy(&send_addr, urb->transfer_buffer, in xhci_queue_bulk_tx()
3683 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, in xhci_queue_bulk_tx()
3690 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, in xhci_queue_bulk_tx()
3695 td->num_trbs++; in xhci_queue_bulk_tx()
3701 --num_sgs; in xhci_queue_bulk_tx()
3702 sent_len -= block_len; in xhci_queue_bulk_tx()
3710 block_len -= sent_len; in xhci_queue_bulk_tx()
3715 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3716 ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3718 urb_priv->td[1].last_trb = ring->enqueue; in xhci_queue_bulk_tx()
3719 urb_priv->td[1].last_trb_seg = ring->enq_seg; in xhci_queue_bulk_tx()
3720 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC; in xhci_queue_bulk_tx()
3721 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); in xhci_queue_bulk_tx()
3722 urb_priv->td[1].num_trbs++; in xhci_queue_bulk_tx()
3726 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3731 /* Caller must have locked xhci->lock */
3732 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_ctrl_tx() argument
3745 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_ctrl_tx()
3747 return -EINVAL; in xhci_queue_ctrl_tx()
3753 if (!urb->setup_packet) in xhci_queue_ctrl_tx()
3754 return -EINVAL; in xhci_queue_ctrl_tx()
3763 if (urb->transfer_buffer_length > 0) in xhci_queue_ctrl_tx()
3765 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3766 ep_index, urb->stream_id, in xhci_queue_ctrl_tx()
3771 urb_priv = urb->hcpriv; in xhci_queue_ctrl_tx()
3772 td = &urb_priv->td[0]; in xhci_queue_ctrl_tx()
3773 td->num_trbs = num_trbs; in xhci_queue_ctrl_tx()
3780 start_trb = &ep_ring->enqueue->generic; in xhci_queue_ctrl_tx()
3781 start_cycle = ep_ring->cycle_state; in xhci_queue_ctrl_tx()
3783 /* Queue setup TRB - see section 6.4.1.2.1 */ in xhci_queue_ctrl_tx()
3785 setup = (struct usb_ctrlrequest *) urb->setup_packet; in xhci_queue_ctrl_tx()
3791 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */ in xhci_queue_ctrl_tx()
3792 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3793 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3794 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3801 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3802 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16, in xhci_queue_ctrl_tx()
3803 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16, in xhci_queue_ctrl_tx()
3815 if (urb->transfer_buffer_length > 0) { in xhci_queue_ctrl_tx()
3820 memcpy(&addr, urb->transfer_buffer, in xhci_queue_ctrl_tx()
3821 urb->transfer_buffer_length); in xhci_queue_ctrl_tx()
3825 addr = (u64) urb->transfer_dma; in xhci_queue_ctrl_tx()
3828 remainder = xhci_td_remainder(xhci, 0, in xhci_queue_ctrl_tx()
3829 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3830 urb->transfer_buffer_length, in xhci_queue_ctrl_tx()
3832 length_field = TRB_LEN(urb->transfer_buffer_length) | in xhci_queue_ctrl_tx()
3835 if (setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3837 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3841 field | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3845 td->last_trb = ep_ring->enqueue; in xhci_queue_ctrl_tx()
3846 td->last_trb_seg = ep_ring->enq_seg; in xhci_queue_ctrl_tx()
3848 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */ in xhci_queue_ctrl_tx()
3850 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN) in xhci_queue_ctrl_tx()
3854 queue_trb(xhci, ep_ring, false, in xhci_queue_ctrl_tx()
3859 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); in xhci_queue_ctrl_tx()
3861 giveback_first_trb(xhci, slot_id, ep_index, 0, in xhci_queue_ctrl_tx()
3872 * zero. Only xHCI 1.0 host controllers support this field.
3874 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, in xhci_get_burst_count() argument
3879 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3882 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_burst_count()
3883 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; in xhci_get_burst_count()
3894 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, in xhci_get_last_burst_packet_count() argument
3900 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3903 if (urb->dev->speed >= USB_SPEED_SUPER) { in xhci_get_last_burst_packet_count()
3905 max_burst = urb->ep->ss_ep_comp.bMaxBurst; in xhci_get_last_burst_packet_count()
3908 * number of packets, but the TLBPC field is zero-based. in xhci_get_last_burst_packet_count()
3912 return residue - 1; in xhci_get_last_burst_packet_count()
3916 return total_packet_count - 1; in xhci_get_last_burst_packet_count()
3926 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, in xhci_get_isoc_frame_id() argument
3932 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3933 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3934 start_frame = urb->start_frame + index * urb->interval; in xhci_get_isoc_frame_id()
3936 start_frame = (urb->start_frame + index * urb->interval) >> 3; in xhci_get_isoc_frame_id()
3946 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
3947 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
3963 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
3971 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n", in xhci_get_isoc_frame_id()
3972 __func__, index, readl(&xhci->run_regs->microframe_index), in xhci_get_isoc_frame_id()
3978 ret = -EINVAL; in xhci_get_isoc_frame_id()
3982 ret = -EINVAL; in xhci_get_isoc_frame_id()
3984 ret = -EINVAL; in xhci_get_isoc_frame_id()
3988 if (ret == -EINVAL || start_frame == start_frame_id) { in xhci_get_isoc_frame_id()
3990 if (urb->dev->speed == USB_SPEED_LOW || in xhci_get_isoc_frame_id()
3991 urb->dev->speed == USB_SPEED_FULL) in xhci_get_isoc_frame_id()
3992 urb->start_frame = start_frame; in xhci_get_isoc_frame_id()
3994 urb->start_frame = start_frame << 3; in xhci_get_isoc_frame_id()
4000 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", in xhci_get_isoc_frame_id()
4003 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); in xhci_get_isoc_frame_id()
4011 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i) in trb_block_event_intr() argument
4013 if (xhci->hci_version < 0x100) in trb_block_event_intr()
4016 if (i == num_tds - 1) in trb_block_event_intr()
4022 if (i && xhci->quirks & XHCI_AVOID_BEI) in trb_block_event_intr()
4023 return !!(i % xhci->isoc_bei_interval); in trb_block_event_intr()
4029 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx() argument
4047 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
4048 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
4050 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx()
4052 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); in xhci_queue_isoc_tx()
4053 return -EINVAL; in xhci_queue_isoc_tx()
4055 start_addr = (u64) urb->transfer_dma; in xhci_queue_isoc_tx()
4056 start_trb = &ep_ring->enqueue->generic; in xhci_queue_isoc_tx()
4057 start_cycle = ep_ring->cycle_state; in xhci_queue_isoc_tx()
4059 urb_priv = urb->hcpriv; in xhci_queue_isoc_tx()
4060 /* Queue the TRBs for each TD, even if they are zero-length */ in xhci_queue_isoc_tx()
4068 addr = start_addr + urb->iso_frame_desc[i].offset; in xhci_queue_isoc_tx()
4069 td_len = urb->iso_frame_desc[i].length; in xhci_queue_isoc_tx()
4071 max_pkt = usb_endpoint_maxp(&urb->ep->desc); in xhci_queue_isoc_tx()
4074 /* A zero-length transfer still involves at least one packet. */ in xhci_queue_isoc_tx()
4077 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); in xhci_queue_isoc_tx()
4078 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, in xhci_queue_isoc_tx()
4083 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
4084 urb->stream_id, trbs_per_td, urb, i, mem_flags); in xhci_queue_isoc_tx()
4090 td = &urb_priv->td[i]; in xhci_queue_isoc_tx()
4091 td->num_trbs = trbs_per_td; in xhci_queue_isoc_tx()
4094 if (!(urb->transfer_flags & URB_ISO_ASAP) && in xhci_queue_isoc_tx()
4095 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
4096 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); in xhci_queue_isoc_tx()
4108 (i ? ep_ring->cycle_state : !start_cycle); in xhci_queue_isoc_tx()
4110 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */ in xhci_queue_isoc_tx()
4111 if (!xep->use_extended_tbc) in xhci_queue_isoc_tx()
4121 ep_ring->cycle_state; in xhci_queue_isoc_tx()
4128 if (j < trbs_per_td - 1) { in xhci_queue_isoc_tx()
4133 td->last_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4134 td->last_trb_seg = ep_ring->enq_seg; in xhci_queue_isoc_tx()
4136 if (trb_block_event_intr(xhci, num_tds, i)) in xhci_queue_isoc_tx()
4145 remainder = xhci_td_remainder(xhci, running_total, in xhci_queue_isoc_tx()
4152 /* xhci 1.1 with ETE uses TD Size field for TBC */ in xhci_queue_isoc_tx()
4153 if (first_trb && xep->use_extended_tbc) in xhci_queue_isoc_tx()
4159 queue_trb(xhci, ep_ring, more_trbs_coming, in xhci_queue_isoc_tx()
4167 td_remain_len -= trb_buff_len; in xhci_queue_isoc_tx()
4172 xhci_err(xhci, "ISOC TD length unmatch\n"); in xhci_queue_isoc_tx()
4173 ret = -EINVAL; in xhci_queue_isoc_tx()
4179 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
4180 xep->next_frame_id = urb->start_frame + num_tds * urb->interval; in xhci_queue_isoc_tx()
4182 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
4183 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
4186 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
4188 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
4194 for (i--; i >= 0; i--) in xhci_queue_isoc_tx()
4195 list_del_init(&urb_priv->td[i].td_list); in xhci_queue_isoc_tx()
4198 * into No-ops with a software-owned cycle bit. That way the hardware in xhci_queue_isoc_tx()
4200 * overwrite them. td->first_trb and td->start_seg are already set. in xhci_queue_isoc_tx()
4202 urb_priv->td[0].last_trb = ep_ring->enqueue; in xhci_queue_isoc_tx()
4204 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
4207 ep_ring->enqueue = urb_priv->td[0].first_trb; in xhci_queue_isoc_tx()
4208 ep_ring->enq_seg = urb_priv->td[0].start_seg; in xhci_queue_isoc_tx()
4209 ep_ring->cycle_state = start_cycle; in xhci_queue_isoc_tx()
4210 ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp; in xhci_queue_isoc_tx()
4211 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb); in xhci_queue_isoc_tx()
4218 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
4219 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
4222 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx_prepare() argument
4234 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
4235 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
4236 ep_ring = xdev->eps[ep_index].ring; in xhci_queue_isoc_tx_prepare()
4237 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
4240 num_tds = urb->number_of_packets; in xhci_queue_isoc_tx_prepare()
4247 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in xhci_queue_isoc_tx_prepare()
4256 check_interval(xhci, urb, ep_ctx); in xhci_queue_isoc_tx_prepare()
4258 /* Calculate the start frame and put it in urb->start_frame. */ in xhci_queue_isoc_tx_prepare()
4259 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
4261 urb->start_frame = xep->next_frame_id; in xhci_queue_isoc_tx_prepare()
4266 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
4272 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
4273 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
4282 if (urb->dev->speed == USB_SPEED_LOW || in xhci_queue_isoc_tx_prepare()
4283 urb->dev->speed == USB_SPEED_FULL) { in xhci_queue_isoc_tx_prepare()
4284 start_frame = roundup(start_frame, urb->interval << 3); in xhci_queue_isoc_tx_prepare()
4285 urb->start_frame = start_frame >> 3; in xhci_queue_isoc_tx_prepare()
4287 start_frame = roundup(start_frame, urb->interval); in xhci_queue_isoc_tx_prepare()
4288 urb->start_frame = start_frame; in xhci_queue_isoc_tx_prepare()
4292 ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free; in xhci_queue_isoc_tx_prepare()
4294 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_isoc_tx_prepare()
4299 /* Generic function for queueing a command TRB on the command ring.
4304 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
4307 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in queue_command() argument
4311 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
4314 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
4315 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
4316 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); in queue_command()
4317 return -ESHUTDOWN; in queue_command()
4323 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
4326 xhci_err(xhci, "ERR: No room for command on command ring\n"); in queue_command()
4328 xhci_err(xhci, "ERR: Reserved TRB counting for " in queue_command()
4333 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
4336 if (list_empty(&xhci->cmd_list)) { in queue_command()
4337 xhci->current_cmd = cmd; in queue_command()
4338 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in queue_command()
4341 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
4343 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
4344 field4 | xhci->cmd_ring->cycle_state); in queue_command()
4349 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_slot_control() argument
4352 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_slot_control()
4357 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_address_device() argument
4360 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_address_device()
4366 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_vendor_command() argument
4369 return queue_command(xhci, cmd, field1, field2, field3, field4, false); in xhci_queue_vendor_command()
4373 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_device() argument
4376 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_device()
4382 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, in xhci_queue_configure_endpoint() argument
4386 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_configure_endpoint()
4393 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_evaluate_context() argument
4396 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_evaluate_context()
4406 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_stop_endpoint() argument
4414 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_stop_endpoint()
4418 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_ep() argument
4429 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_ep()