Lines Matching refs:xhci
139 static void next_trb(struct xhci_hcd *xhci, in next_trb() argument
156 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) in inc_deq() argument
204 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, in inc_enq() argument
234 (xhci->quirks & XHCI_AMD_0x96_HOST)) && in inc_enq()
235 !xhci_link_trb_quirk(xhci)) { in inc_enq()
259 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring, in room_on_ring() argument
277 void xhci_ring_cmd_db(struct xhci_hcd *xhci) in xhci_ring_cmd_db() argument
279 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) in xhci_ring_cmd_db()
282 xhci_dbg(xhci, "// Ding dong!\n"); in xhci_ring_cmd_db()
283 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
285 readl(&xhci->dba->doorbell[0]); in xhci_ring_cmd_db()
288 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay) in xhci_mod_cmd_timer() argument
290 return mod_delayed_work(system_wq, &xhci->cmd_timer, delay); in xhci_mod_cmd_timer()
293 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) in xhci_next_queued_cmd() argument
295 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, in xhci_next_queued_cmd()
304 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, in xhci_handle_stopped_cmd_ring() argument
310 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { in xhci_handle_stopped_cmd_ring()
317 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", in xhci_handle_stopped_cmd_ring()
328 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; in xhci_handle_stopped_cmd_ring()
331 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && in xhci_handle_stopped_cmd_ring()
332 !(xhci->xhc_state & XHCI_STATE_DYING)) { in xhci_handle_stopped_cmd_ring()
333 xhci->current_cmd = cur_cmd; in xhci_handle_stopped_cmd_ring()
334 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in xhci_handle_stopped_cmd_ring()
335 xhci_ring_cmd_db(xhci); in xhci_handle_stopped_cmd_ring()
340 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) in xhci_abort_cmd_ring() argument
345 xhci_dbg(xhci, "Abort command ring\n"); in xhci_abort_cmd_ring()
347 reinit_completion(&xhci->cmd_ring_stop_completion); in xhci_abort_cmd_ring()
349 temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
350 xhci_write_64(xhci, temp_64 | CMD_RING_ABORT, in xhci_abort_cmd_ring()
351 &xhci->op_regs->cmd_ring); in xhci_abort_cmd_ring()
359 ret = xhci_handshake(&xhci->op_regs->cmd_ring, in xhci_abort_cmd_ring()
362 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); in xhci_abort_cmd_ring()
363 xhci_halt(xhci); in xhci_abort_cmd_ring()
364 xhci_hc_died(xhci); in xhci_abort_cmd_ring()
373 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_abort_cmd_ring()
374 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, in xhci_abort_cmd_ring()
376 spin_lock_irqsave(&xhci->lock, flags); in xhci_abort_cmd_ring()
378 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); in xhci_abort_cmd_ring()
379 xhci_cleanup_command_queue(xhci); in xhci_abort_cmd_ring()
381 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); in xhci_abort_cmd_ring()
386 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, in xhci_ring_ep_doorbell() argument
391 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; in xhci_ring_ep_doorbell()
392 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_ring_ep_doorbell()
411 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, in ring_doorbell_for_active_rings() argument
418 ep = &xhci->devs[slot_id]->eps[ep_index]; in ring_doorbell_for_active_rings()
423 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); in ring_doorbell_for_active_rings()
431 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, in ring_doorbell_for_active_rings()
440 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, in xhci_triad_to_transfer_ring() argument
446 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_triad_to_transfer_ring()
452 xhci_warn(xhci, in xhci_triad_to_transfer_ring()
462 xhci_warn(xhci, in xhci_triad_to_transfer_ring()
479 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, in xhci_get_hw_deq() argument
492 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_get_hw_deq()
514 void xhci_find_new_dequeue_state(struct xhci_hcd *xhci, in xhci_find_new_dequeue_state() argument
519 struct xhci_virt_device *dev = xhci->devs[slot_id]; in xhci_find_new_dequeue_state()
529 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, in xhci_find_new_dequeue_state()
532 xhci_warn(xhci, "WARN can't find new dequeue state " in xhci_find_new_dequeue_state()
538 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_find_new_dequeue_state()
541 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); in xhci_find_new_dequeue_state()
567 next_trb(xhci, ep_ring, &new_seg, &new_deq); in xhci_find_new_dequeue_state()
571 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); in xhci_find_new_dequeue_state()
583 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_find_new_dequeue_state()
586 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_find_new_dequeue_state()
590 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_find_new_dequeue_state()
599 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in td_to_noop() argument
615 next_trb(xhci, ep_ring, &seg, &trb); in td_to_noop()
619 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci, in xhci_stop_watchdog_timer_in_irq() argument
631 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, in xhci_giveback_urb_in_irq() argument
639 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; in xhci_giveback_urb_in_irq()
640 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_giveback_urb_in_irq()
641 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_giveback_urb_in_irq()
647 spin_unlock(&xhci->lock); in xhci_giveback_urb_in_irq()
650 spin_lock(&xhci->lock); in xhci_giveback_urb_in_irq()
653 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, in xhci_unmap_td_bounce_buffer() argument
656 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_unmap_td_bounce_buffer()
688 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_stop_ep() argument
702 if (!xhci->devs[slot_id]) in xhci_handle_cmd_stop_ep()
703 xhci_warn(xhci, "Stop endpoint command " in xhci_handle_cmd_stop_ep()
712 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_stop_ep()
713 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_handle_cmd_stop_ep()
716 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_handle_cmd_stop_ep()
721 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
722 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
732 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_stop_ep()
736 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb); in xhci_handle_cmd_stop_ep()
749 xhci_warn(xhci, "WARN Cancelled URB %p " in xhci_handle_cmd_stop_ep()
759 hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index, in xhci_handle_cmd_stop_ep()
763 if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb, in xhci_handle_cmd_stop_ep()
765 xhci_find_new_dequeue_state(xhci, slot_id, ep_index, in xhci_handle_cmd_stop_ep()
769 td_to_noop(xhci, ep_ring, cur_td, false); in xhci_handle_cmd_stop_ep()
781 xhci_stop_watchdog_timer_in_irq(xhci, ep); in xhci_handle_cmd_stop_ep()
785 xhci_queue_new_dequeue_state(xhci, slot_id, ep_index, in xhci_handle_cmd_stop_ep()
787 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_stop_ep()
790 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_stop_ep()
808 ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb); in xhci_handle_cmd_stop_ep()
809 xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td); in xhci_handle_cmd_stop_ep()
812 xhci_giveback_urb_in_irq(xhci, cur_td, 0); in xhci_handle_cmd_stop_ep()
817 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_handle_cmd_stop_ep()
824 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_kill_ring_urbs() argument
835 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); in xhci_kill_ring_urbs()
839 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_ring_urbs()
843 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, in xhci_kill_endpoint_urbs() argument
851 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_kill_endpoint_urbs()
862 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
865 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
871 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_kill_endpoint_urbs()
874 xhci_kill_ring_urbs(xhci, ring); in xhci_kill_endpoint_urbs()
883 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); in xhci_kill_endpoint_urbs()
896 void xhci_hc_died(struct xhci_hcd *xhci) in xhci_hc_died() argument
900 if (xhci->xhc_state & XHCI_STATE_DYING) in xhci_hc_died()
903 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); in xhci_hc_died()
904 xhci->xhc_state |= XHCI_STATE_DYING; in xhci_hc_died()
906 xhci_cleanup_command_queue(xhci); in xhci_hc_died()
909 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_hc_died()
910 if (!xhci->devs[i]) in xhci_hc_died()
913 xhci_kill_endpoint_urbs(xhci, i, j); in xhci_hc_died()
917 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) in xhci_hc_died()
918 usb_hc_died(xhci_to_hcd(xhci)); in xhci_hc_died()
941 struct xhci_hcd *xhci = ep->xhci; in xhci_stop_endpoint_command_watchdog() local
944 spin_lock_irqsave(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
949 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
950 xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit"); in xhci_stop_endpoint_command_watchdog()
954 xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n"); in xhci_stop_endpoint_command_watchdog()
957 xhci_halt(xhci); in xhci_stop_endpoint_command_watchdog()
964 xhci_hc_died(xhci); in xhci_stop_endpoint_command_watchdog()
966 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_stop_endpoint_command_watchdog()
967 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_stop_endpoint_command_watchdog()
971 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci, in update_ring_for_set_deq_completion() argument
1012 xhci_dbg(xhci, "Unable to find new dequeue pointer\n"); in update_ring_for_set_deq_completion()
1024 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_set_deq() argument
1037 dev = xhci->devs[slot_id]; in xhci_handle_cmd_set_deq()
1042 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", in xhci_handle_cmd_set_deq()
1048 ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index); in xhci_handle_cmd_set_deq()
1049 slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx); in xhci_handle_cmd_set_deq()
1059 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); in xhci_handle_cmd_set_deq()
1062 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); in xhci_handle_cmd_set_deq()
1066 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1071 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", in xhci_handle_cmd_set_deq()
1075 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", in xhci_handle_cmd_set_deq()
1095 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_handle_cmd_set_deq()
1102 update_ring_for_set_deq_completion(xhci, dev, in xhci_handle_cmd_set_deq()
1105 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); in xhci_handle_cmd_set_deq()
1106 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", in xhci_handle_cmd_set_deq()
1116 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_set_deq()
1119 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_ep() argument
1127 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_ep()
1128 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); in xhci_handle_cmd_reset_ep()
1134 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, in xhci_handle_cmd_reset_ep()
1141 if (xhci->quirks & XHCI_RESET_EP_QUIRK) { in xhci_handle_cmd_reset_ep()
1144 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_handle_cmd_reset_ep()
1148 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_reset_ep()
1150 xhci_queue_configure_endpoint(xhci, command, in xhci_handle_cmd_reset_ep()
1151 xhci->devs[slot_id]->in_ctx->dma, slot_id, in xhci_handle_cmd_reset_ep()
1153 xhci_ring_cmd_db(xhci); in xhci_handle_cmd_reset_ep()
1156 xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED; in xhci_handle_cmd_reset_ep()
1160 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_enable_slot() argument
1169 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_disable_slot() argument
1174 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_disable_slot()
1178 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); in xhci_handle_cmd_disable_slot()
1181 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) in xhci_handle_cmd_disable_slot()
1183 xhci_free_device_endpoint_resources(xhci, virt_dev, true); in xhci_handle_cmd_disable_slot()
1184 xhci_free_virt_device(xhci, slot_id); in xhci_handle_cmd_disable_slot()
1187 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_config_ep() argument
1205 virt_dev = xhci->devs[slot_id]; in xhci_handle_cmd_config_ep()
1208 xhci_warn(xhci, "Could not get input context, bad type.\n"); in xhci_handle_cmd_config_ep()
1217 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); in xhci_handle_cmd_config_ep()
1226 if (xhci->quirks & XHCI_RESET_EP_QUIRK && in xhci_handle_cmd_config_ep()
1232 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_config_ep()
1238 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); in xhci_handle_cmd_config_ep()
1244 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) in xhci_handle_cmd_addr_dev() argument
1249 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_addr_dev()
1250 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_addr_dev()
1254 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id, in xhci_handle_cmd_reset_dev() argument
1260 vdev = xhci->devs[slot_id]; in xhci_handle_cmd_reset_dev()
1261 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); in xhci_handle_cmd_reset_dev()
1264 xhci_dbg(xhci, "Completed reset device command.\n"); in xhci_handle_cmd_reset_dev()
1265 if (!xhci->devs[slot_id]) in xhci_handle_cmd_reset_dev()
1266 xhci_warn(xhci, "Reset device command completion " in xhci_handle_cmd_reset_dev()
1270 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, in xhci_handle_cmd_nec_get_fw() argument
1273 if (!(xhci->quirks & XHCI_NEC_HOST)) { in xhci_handle_cmd_nec_get_fw()
1274 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); in xhci_handle_cmd_nec_get_fw()
1277 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, in xhci_handle_cmd_nec_get_fw()
1295 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) in xhci_cleanup_command_queue() argument
1298 xhci->current_cmd = NULL; in xhci_cleanup_command_queue()
1299 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) in xhci_cleanup_command_queue()
1305 struct xhci_hcd *xhci; in xhci_handle_command_timeout() local
1309 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); in xhci_handle_command_timeout()
1311 spin_lock_irqsave(&xhci->lock, flags); in xhci_handle_command_timeout()
1317 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { in xhci_handle_command_timeout()
1318 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1322 xhci->current_cmd->status = COMP_COMMAND_ABORTED; in xhci_handle_command_timeout()
1325 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_handle_command_timeout()
1327 xhci_hc_died(xhci); in xhci_handle_command_timeout()
1331 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && in xhci_handle_command_timeout()
1334 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; in xhci_handle_command_timeout()
1335 xhci_dbg(xhci, "Command timeout\n"); in xhci_handle_command_timeout()
1336 xhci_abort_cmd_ring(xhci, flags); in xhci_handle_command_timeout()
1341 if (xhci->xhc_state & XHCI_STATE_REMOVING) { in xhci_handle_command_timeout()
1342 xhci_dbg(xhci, "host removed, ring start fail?\n"); in xhci_handle_command_timeout()
1343 xhci_cleanup_command_queue(xhci); in xhci_handle_command_timeout()
1349 xhci_dbg(xhci, "Command timeout on stopped ring\n"); in xhci_handle_command_timeout()
1350 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); in xhci_handle_command_timeout()
1353 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_handle_command_timeout()
1357 static void handle_cmd_completion(struct xhci_hcd *xhci, in handle_cmd_completion() argument
1369 cmd_trb = xhci->cmd_ring->dequeue; in handle_cmd_completion()
1371 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); in handle_cmd_completion()
1373 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, in handle_cmd_completion()
1380 xhci_warn(xhci, in handle_cmd_completion()
1385 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); in handle_cmd_completion()
1387 cancel_delayed_work(&xhci->cmd_timer); in handle_cmd_completion()
1393 complete_all(&xhci->cmd_ring_stop_completion); in handle_cmd_completion()
1397 if (cmd->command_trb != xhci->cmd_ring->dequeue) { in handle_cmd_completion()
1398 xhci_err(xhci, in handle_cmd_completion()
1410 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; in handle_cmd_completion()
1412 if (xhci->current_cmd == cmd) in handle_cmd_completion()
1413 xhci->current_cmd = NULL; in handle_cmd_completion()
1421 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code); in handle_cmd_completion()
1424 xhci_handle_cmd_disable_slot(xhci, slot_id); in handle_cmd_completion()
1428 xhci_handle_cmd_config_ep(xhci, slot_id, event, in handle_cmd_completion()
1434 xhci_handle_cmd_addr_dev(xhci, slot_id); in handle_cmd_completion()
1440 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event); in handle_cmd_completion()
1445 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1455 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); in handle_cmd_completion()
1463 xhci_handle_cmd_reset_dev(xhci, slot_id, event); in handle_cmd_completion()
1466 xhci_handle_cmd_nec_get_fw(xhci, event); in handle_cmd_completion()
1470 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); in handle_cmd_completion()
1475 if (!list_is_singular(&xhci->cmd_list)) { in handle_cmd_completion()
1476 xhci->current_cmd = list_first_entry(&cmd->cmd_list, in handle_cmd_completion()
1478 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in handle_cmd_completion()
1479 } else if (xhci->current_cmd == cmd) { in handle_cmd_completion()
1480 xhci->current_cmd = NULL; in handle_cmd_completion()
1486 inc_deq(xhci, xhci->cmd_ring); in handle_cmd_completion()
1489 static void handle_vendor_event(struct xhci_hcd *xhci, in handle_vendor_event() argument
1495 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); in handle_vendor_event()
1496 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) in handle_vendor_event()
1497 handle_cmd_completion(xhci, &event->event_cmd); in handle_vendor_event()
1500 static void handle_device_notification(struct xhci_hcd *xhci, in handle_device_notification() argument
1507 if (!xhci->devs[slot_id]) { in handle_device_notification()
1508 xhci_warn(xhci, "Device Notification event for " in handle_device_notification()
1513 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", in handle_device_notification()
1515 udev = xhci->devs[slot_id]->udev; in handle_device_notification()
1520 static void handle_port_status(struct xhci_hcd *xhci, in handle_port_status() argument
1535 xhci_warn(xhci, in handle_port_status()
1539 xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id); in handle_port_status()
1541 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); in handle_port_status()
1543 xhci_warn(xhci, "Invalid port id %d\n", port_id); in handle_port_status()
1544 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1548 port = &xhci->hw_ports[port_id - 1]; in handle_port_status()
1550 xhci_warn(xhci, "Event for invalid port %u\n", port_id); in handle_port_status()
1556 bus_state = &xhci->bus_state[hcd_index(hcd)]; in handle_port_status()
1563 xhci_dbg(xhci, "resume root hub\n"); in handle_port_status()
1571 xhci_dbg(xhci, "port resume event for port %d\n", port_id); in handle_port_status()
1573 cmd_reg = readl(&xhci->op_regs->command); in handle_port_status()
1575 xhci_warn(xhci, "xHC is not running.\n"); in handle_port_status()
1580 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); in handle_port_status()
1586 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
1587 xhci_set_link_state(xhci, port, XDEV_U0); in handle_port_status()
1594 xhci_dbg(xhci, "resume HS port %d\n", port_id); in handle_port_status()
1611 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); in handle_port_status()
1619 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); in handle_port_status()
1620 if (slot_id && xhci->devs[slot_id]) in handle_port_status()
1621 xhci_ring_device(xhci, slot_id); in handle_port_status()
1624 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
1646 xhci_test_and_clear_bit(xhci, port, PORT_PLC); in handle_port_status()
1650 inc_deq(xhci, xhci->event_ring); in handle_port_status()
1666 xhci_dbg(xhci, "%s: starting port polling.\n", __func__); in handle_port_status()
1668 spin_unlock(&xhci->lock); in handle_port_status()
1671 spin_lock(&xhci->lock); in handle_port_status()
1680 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci, in trb_in_td() argument
1705 xhci_warn(xhci, in trb_in_td()
1741 static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci, in xhci_cleanup_halted_endpoint() argument
1746 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_cleanup_halted_endpoint()
1748 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_cleanup_halted_endpoint()
1754 xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); in xhci_cleanup_halted_endpoint()
1758 xhci_cleanup_stalled_ring(xhci, ep_index, stream_id, td); in xhci_cleanup_halted_endpoint()
1760 xhci_ring_cmd_db(xhci); in xhci_cleanup_halted_endpoint()
1769 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci, in xhci_requires_manual_halt_cleanup() argument
1789 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) in xhci_is_vendor_info_code() argument
1795 xhci_dbg(xhci, "Vendor defined info completion code %u\n", in xhci_is_vendor_info_code()
1797 xhci_dbg(xhci, "Treating code as success.\n"); in xhci_is_vendor_info_code()
1803 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, in xhci_td_cleanup() argument
1812 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); in xhci_td_cleanup()
1820 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", in xhci_td_cleanup()
1836 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", in xhci_td_cleanup()
1843 xhci_giveback_urb_in_irq(xhci, td, *status); in xhci_td_cleanup()
1849 static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td, in finish_td() argument
1861 xdev = xhci->devs[slot_id]; in finish_td()
1864 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in finish_td()
1877 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in finish_td()
1884 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, in finish_td()
1889 inc_deq(xhci, ep_ring); in finish_td()
1890 inc_deq(xhci, ep_ring); in finish_td()
1893 return xhci_td_cleanup(xhci, td, ep_ring, status); in finish_td()
1897 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, in sum_trb_lengths() argument
1904 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) { in sum_trb_lengths()
1914 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td, in process_ctrl_td() argument
1928 xdev = xhci->devs[slot_id]; in process_ctrl_td()
1930 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in process_ctrl_td()
1938 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", in process_ctrl_td()
1952 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); in process_ctrl_td()
1967 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", in process_ctrl_td()
1974 if (!xhci_requires_manual_halt_cleanup(xhci, in process_ctrl_td()
1977 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", in process_ctrl_td()
2001 xhci_dbg(xhci, "Waiting for status stage event\n"); in process_ctrl_td()
2010 return finish_td(xhci, td, event, ep, status); in process_ctrl_td()
2016 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in process_isoc_td() argument
2045 if (xhci->quirks & XHCI_TRUST_TX_LENGTH) in process_isoc_td()
2090 frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) + in process_isoc_td()
2097 return finish_td(xhci, td, event, ep, status); in process_isoc_td()
2100 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, in skip_isoc_td() argument
2122 inc_deq(xhci, ep_ring); in skip_isoc_td()
2123 inc_deq(xhci, ep_ring); in skip_isoc_td()
2125 return xhci_td_cleanup(xhci, td, ep_ring, status); in skip_isoc_td()
2131 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td, in process_bulk_intr_td() argument
2149 xhci_warn(xhci, "WARN Successful completion on short TX\n"); in process_bulk_intr_td()
2150 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2157 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", in process_bulk_intr_td()
2179 sum_trb_lengths(xhci, ep_ring, ep_trb) + in process_bulk_intr_td()
2183 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", in process_bulk_intr_td()
2187 return finish_td(xhci, td, event, ep, status); in process_bulk_intr_td()
2195 static int handle_tx_event(struct xhci_hcd *xhci, in handle_tx_event() argument
2219 xdev = xhci->devs[slot_id]; in handle_tx_event()
2221 xhci_err(xhci, "ERROR Transfer event pointed to bad slot %u\n", in handle_tx_event()
2228 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in handle_tx_event()
2231 xhci_err(xhci, in handle_tx_event()
2244 xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0, in handle_tx_event()
2251 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n", in handle_tx_event()
2271 if (xhci->quirks & XHCI_TRUST_TX_LENGTH) in handle_tx_event()
2274 xhci_warn_ratelimited(xhci, in handle_tx_event()
2281 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", in handle_tx_event()
2285 xhci_dbg(xhci, in handle_tx_event()
2290 xhci_dbg(xhci, in handle_tx_event()
2296 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, in handle_tx_event()
2303 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", in handle_tx_event()
2308 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", in handle_tx_event()
2314 xhci_warn(xhci, in handle_tx_event()
2321 xhci_warn(xhci, in handle_tx_event()
2327 xhci_warn(xhci, in handle_tx_event()
2332 xhci_warn(xhci, in handle_tx_event()
2342 xhci_dbg(xhci, "underrun event on endpoint\n"); in handle_tx_event()
2344 xhci_dbg(xhci, "Underrun Event for slot %d ep %d " in handle_tx_event()
2350 xhci_dbg(xhci, "overrun event on endpoint\n"); in handle_tx_event()
2352 xhci_dbg(xhci, "Overrun Event for slot %d ep %d " in handle_tx_event()
2365 xhci_dbg(xhci, in handle_tx_event()
2371 xhci_dbg(xhci, in handle_tx_event()
2378 xhci_warn(xhci, in handle_tx_event()
2384 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { in handle_tx_event()
2388 xhci_warn(xhci, in handle_tx_event()
2410 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n", in handle_tx_event()
2416 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2425 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n", in handle_tx_event()
2436 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue, in handle_tx_event()
2459 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && in handle_tx_event()
2465 xhci_err(xhci, in handle_tx_event()
2470 trb_in_td(xhci, ep_ring->deq_seg, in handle_tx_event()
2476 skip_isoc_td(xhci, td, event, ep, &status); in handle_tx_event()
2485 xhci_dbg(xhci, in handle_tx_event()
2506 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, in handle_tx_event()
2508 xhci_cleanup_halted_endpoint(xhci, slot_id, in handle_tx_event()
2517 process_ctrl_td(xhci, td, ep_trb, event, ep, &status); in handle_tx_event()
2519 process_isoc_td(xhci, td, ep_trb, event, ep, &status); in handle_tx_event()
2521 process_bulk_intr_td(xhci, td, ep_trb, event, ep, in handle_tx_event()
2533 inc_deq(xhci, xhci->event_ring); in handle_tx_event()
2546 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", in handle_tx_event()
2548 xhci->event_ring->deq_seg, in handle_tx_event()
2549 xhci->event_ring->dequeue), in handle_tx_event()
2563 static int xhci_handle_event(struct xhci_hcd *xhci) in xhci_handle_event() argument
2570 if (!xhci->event_ring || !xhci->event_ring->dequeue) { in xhci_handle_event()
2571 xhci_err(xhci, "ERROR event ring not ready\n"); in xhci_handle_event()
2575 event = xhci->event_ring->dequeue; in xhci_handle_event()
2578 xhci->event_ring->cycle_state) in xhci_handle_event()
2581 trace_xhci_handle_event(xhci->event_ring, &event->generic); in xhci_handle_event()
2591 handle_cmd_completion(xhci, &event->event_cmd); in xhci_handle_event()
2594 handle_port_status(xhci, event); in xhci_handle_event()
2598 ret = handle_tx_event(xhci, &event->trans_event); in xhci_handle_event()
2603 handle_device_notification(xhci, event); in xhci_handle_event()
2608 handle_vendor_event(xhci, event); in xhci_handle_event()
2610 xhci_warn(xhci, "ERROR unknown event type %d\n", in xhci_handle_event()
2617 if (xhci->xhc_state & XHCI_STATE_DYING) { in xhci_handle_event()
2618 xhci_dbg(xhci, "xHCI host dying, returning from " in xhci_handle_event()
2625 inc_deq(xhci, xhci->event_ring); in xhci_handle_event()
2640 struct xhci_hcd *xhci = hcd_to_xhci(hcd); in xhci_irq() local
2648 spin_lock_irqsave(&xhci->lock, flags); in xhci_irq()
2650 status = readl(&xhci->op_regs->status); in xhci_irq()
2652 xhci_hc_died(xhci); in xhci_irq()
2661 xhci_warn(xhci, "WARNING: Host System Error\n"); in xhci_irq()
2662 xhci_halt(xhci); in xhci_irq()
2673 writel(status, &xhci->op_regs->status); in xhci_irq()
2677 irq_pending = readl(&xhci->ir_set->irq_pending); in xhci_irq()
2679 writel(irq_pending, &xhci->ir_set->irq_pending); in xhci_irq()
2682 if (xhci->xhc_state & XHCI_STATE_DYING || in xhci_irq()
2683 xhci->xhc_state & XHCI_STATE_HALTED) { in xhci_irq()
2684 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. " in xhci_irq()
2689 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_irq()
2690 xhci_write_64(xhci, temp_64 | ERST_EHB, in xhci_irq()
2691 &xhci->ir_set->erst_dequeue); in xhci_irq()
2696 event_ring_deq = xhci->event_ring->dequeue; in xhci_irq()
2700 while (xhci_handle_event(xhci) > 0) {} in xhci_irq()
2702 temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_irq()
2704 if (event_ring_deq != xhci->event_ring->dequeue) { in xhci_irq()
2705 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_irq()
2706 xhci->event_ring->dequeue); in xhci_irq()
2708 xhci_warn(xhci, "WARN something wrong with SW event " in xhci_irq()
2717 xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue); in xhci_irq()
2721 spin_unlock_irqrestore(&xhci->lock, flags); in xhci_irq()
2740 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, in queue_trb() argument
2754 inc_enq(xhci, ring, more_trbs_coming); in queue_trb()
2761 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, in prepare_ring() argument
2773 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); in prepare_ring()
2776 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); in prepare_ring()
2781 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); in prepare_ring()
2786 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); in prepare_ring()
2795 if (room_on_ring(xhci, ep_ring, num_trbs)) in prepare_ring()
2798 if (ep_ring == xhci->cmd_ring) { in prepare_ring()
2799 xhci_err(xhci, "Do not support expand command ring\n"); in prepare_ring()
2803 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in prepare_ring()
2806 if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed, in prepare_ring()
2808 xhci_err(xhci, "Ring expansion failed\n"); in prepare_ring()
2817 if (!xhci_link_trb_quirk(xhci) && in prepare_ring()
2819 (xhci->quirks & XHCI_AMD_0x96_HOST))) in prepare_ring()
2839 static int prepare_transfer(struct xhci_hcd *xhci, in prepare_transfer() argument
2852 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in prepare_transfer()
2856 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", in prepare_transfer()
2861 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in prepare_transfer()
2945 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, in giveback_first_trb() argument
2958 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); in giveback_first_trb()
2961 static void check_interval(struct xhci_hcd *xhci, struct urb *urb, in check_interval() argument
2997 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_intr_tx() argument
3002 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); in xhci_queue_intr_tx()
3003 check_interval(xhci, urb, ep_ctx); in xhci_queue_intr_tx()
3005 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_intr_tx()
3028 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, in xhci_td_remainder() argument
3035 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) in xhci_td_remainder()
3044 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) in xhci_td_remainder()
3055 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, in xhci_align_td() argument
3058 struct device *dev = xhci_to_hcd(xhci)->self.controller; in xhci_align_td()
3070 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", in xhci_align_td()
3076 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3103 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); in xhci_align_td()
3110 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); in xhci_align_td()
3116 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_bulk_tx() argument
3134 ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_bulk_tx()
3151 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3200 if (xhci_align_td(xhci, urb, enqd_len, in xhci_queue_bulk_tx()
3221 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, in xhci_queue_bulk_tx()
3228 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, in xhci_queue_bulk_tx()
3253 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_bulk_tx()
3258 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); in xhci_queue_bulk_tx()
3262 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_bulk_tx()
3268 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_ctrl_tx() argument
3281 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); in xhci_queue_ctrl_tx()
3301 ret = prepare_transfer(xhci, xhci->devs[slot_id], in xhci_queue_ctrl_tx()
3327 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { in xhci_queue_ctrl_tx()
3336 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3353 remainder = xhci_td_remainder(xhci, 0, in xhci_queue_ctrl_tx()
3362 queue_trb(xhci, ep_ring, true, in xhci_queue_ctrl_tx()
3378 queue_trb(xhci, ep_ring, false, in xhci_queue_ctrl_tx()
3385 giveback_first_trb(xhci, slot_id, ep_index, 0, in xhci_queue_ctrl_tx()
3398 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, in xhci_get_burst_count() argument
3403 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) in xhci_get_burst_count()
3418 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, in xhci_get_last_burst_packet_count() argument
3424 if (xhci->hci_version < 0x100) in xhci_get_last_burst_packet_count()
3450 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, in xhci_get_isoc_frame_id() argument
3470 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_get_isoc_frame_id()
3471 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_get_isoc_frame_id()
3487 current_frame_id = readl(&xhci->run_regs->microframe_index); in xhci_get_isoc_frame_id()
3495 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n", in xhci_get_isoc_frame_id()
3496 __func__, index, readl(&xhci->run_regs->microframe_index), in xhci_get_isoc_frame_id()
3524 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", in xhci_get_isoc_frame_id()
3527 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); in xhci_get_isoc_frame_id()
3535 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx() argument
3553 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx()
3554 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; in xhci_queue_isoc_tx()
3558 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); in xhci_queue_isoc_tx()
3583 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); in xhci_queue_isoc_tx()
3584 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, in xhci_queue_isoc_tx()
3589 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, in xhci_queue_isoc_tx()
3601 HCC_CFC(xhci->hcc_params)) { in xhci_queue_isoc_tx()
3602 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); in xhci_queue_isoc_tx()
3642 if (xhci->hci_version >= 0x100 && in xhci_queue_isoc_tx()
3643 !(xhci->quirks & XHCI_AVOID_BEI) && in xhci_queue_isoc_tx()
3653 remainder = xhci_td_remainder(xhci, running_total, in xhci_queue_isoc_tx()
3667 queue_trb(xhci, ep_ring, more_trbs_coming, in xhci_queue_isoc_tx()
3680 xhci_err(xhci, "ISOC TD length unmatch\n"); in xhci_queue_isoc_tx()
3687 if (HCC_CFC(xhci->hcc_params)) in xhci_queue_isoc_tx()
3690 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { in xhci_queue_isoc_tx()
3691 if (xhci->quirks & XHCI_AMD_PLL_FIX) in xhci_queue_isoc_tx()
3694 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; in xhci_queue_isoc_tx()
3696 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, in xhci_queue_isoc_tx()
3712 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); in xhci_queue_isoc_tx()
3730 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, in xhci_queue_isoc_tx_prepare() argument
3742 xdev = xhci->devs[slot_id]; in xhci_queue_isoc_tx_prepare()
3743 xep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_isoc_tx_prepare()
3745 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); in xhci_queue_isoc_tx_prepare()
3755 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), in xhci_queue_isoc_tx_prepare()
3764 check_interval(xhci, urb, ep_ctx); in xhci_queue_isoc_tx_prepare()
3767 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { in xhci_queue_isoc_tx_prepare()
3774 start_frame = readl(&xhci->run_regs->microframe_index); in xhci_queue_isoc_tx_prepare()
3780 ist = HCS_IST(xhci->hcs_params2) & 0x7; in xhci_queue_isoc_tx_prepare()
3781 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) in xhci_queue_isoc_tx_prepare()
3802 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); in xhci_queue_isoc_tx_prepare()
3815 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in queue_command() argument
3819 int reserved_trbs = xhci->cmd_ring_reserved_trbs; in queue_command()
3822 if ((xhci->xhc_state & XHCI_STATE_DYING) || in queue_command()
3823 (xhci->xhc_state & XHCI_STATE_HALTED)) { in queue_command()
3824 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); in queue_command()
3831 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, in queue_command()
3834 xhci_err(xhci, "ERR: No room for command on command ring\n"); in queue_command()
3836 xhci_err(xhci, "ERR: Reserved TRB counting for " in queue_command()
3841 cmd->command_trb = xhci->cmd_ring->enqueue; in queue_command()
3844 if (list_empty(&xhci->cmd_list)) { in queue_command()
3845 xhci->current_cmd = cmd; in queue_command()
3846 xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT); in queue_command()
3849 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); in queue_command()
3851 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, in queue_command()
3852 field4 | xhci->cmd_ring->cycle_state); in queue_command()
3857 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_slot_control() argument
3860 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_slot_control()
3865 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_address_device() argument
3868 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_address_device()
3874 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_vendor_command() argument
3877 return queue_command(xhci, cmd, field1, field2, field3, field4, false); in xhci_queue_vendor_command()
3881 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_device() argument
3884 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_device()
3890 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, in xhci_queue_configure_endpoint() argument
3894 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_configure_endpoint()
3901 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_evaluate_context() argument
3904 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), in xhci_queue_evaluate_context()
3914 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_stop_endpoint() argument
3922 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_stop_endpoint()
3927 void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci, in xhci_queue_new_dequeue_state() argument
3941 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, in xhci_queue_new_dequeue_state()
3953 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n"); in xhci_queue_new_dequeue_state()
3954 xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n", in xhci_queue_new_dequeue_state()
3958 ep = &xhci->devs[slot_id]->eps[ep_index]; in xhci_queue_new_dequeue_state()
3960 xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n"); in xhci_queue_new_dequeue_state()
3961 xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n"); in xhci_queue_new_dequeue_state()
3966 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); in xhci_queue_new_dequeue_state()
3974 ret = queue_command(xhci, cmd, in xhci_queue_new_dequeue_state()
3979 xhci_free_command(xhci, cmd); in xhci_queue_new_dequeue_state()
3991 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, in xhci_queue_reset_ep() argument
4002 return queue_command(xhci, cmd, 0, 0, 0, in xhci_queue_reset_ep()