1 // SPDX-License-Identifier: GPL-2.0
3  * xHCI host controller driver
19 #include <linux/dma-mapping.h>
21 #include "xhci.h"
22 #include "xhci-trace.h"
23 #include "xhci-debugfs.h"
24 #include "xhci-dbgcap.h"
42 	struct xhci_segment *seg = ring->first_seg;  in td_on_ring()
44 	if (!td || !td->start_seg)  in td_on_ring()
47 		if (seg == td->start_seg)  in td_on_ring()
49 		seg = seg->next;  in td_on_ring()
50 	} while (seg && seg != ring->first_seg);  in td_on_ring()
56  * xhci_handshake - spin reading hc until handshake completes or fails
66  * hardware flakeout), or the register reads as all-ones (hardware removed).
78 		return -ENODEV;  in xhci_handshake()
84  * Disable interrupts and begin the xHCI halting process.
86 void xhci_quiesce(struct xhci_hcd *xhci)  in xhci_quiesce()  argument
93 	halted = readl(&xhci->op_regs->status) & STS_HALT;  in xhci_quiesce()
97 	cmd = readl(&xhci->op_regs->command);  in xhci_quiesce()
99 	writel(cmd, &xhci->op_regs->command);  in xhci_quiesce()
110 int xhci_halt(struct xhci_hcd *xhci)  in xhci_halt()  argument
114 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");  in xhci_halt()
115 	xhci_quiesce(xhci);  in xhci_halt()
117 	ret = xhci_handshake(&xhci->op_regs->status,  in xhci_halt()
120 		xhci_warn(xhci, "Host halt failed, %d\n", ret);  in xhci_halt()
124 	xhci->xhc_state |= XHCI_STATE_HALTED;  in xhci_halt()
125 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;  in xhci_halt()
133 int xhci_start(struct xhci_hcd *xhci)  in xhci_start()  argument
138 	temp = readl(&xhci->op_regs->command);  in xhci_start()
140 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",  in xhci_start()
142 	writel(temp, &xhci->op_regs->command);  in xhci_start()
148 	ret = xhci_handshake(&xhci->op_regs->status,  in xhci_start()
150 	if (ret == -ETIMEDOUT)  in xhci_start()
151 		xhci_err(xhci, "Host took too long to start, "  in xhci_start()
156 		xhci->xhc_state = 0;  in xhci_start()
157 		xhci->run_graceperiod = jiffies + msecs_to_jiffies(500);  in xhci_start()
170 int xhci_reset(struct xhci_hcd *xhci, u64 timeout_us)  in xhci_reset()  argument
176 	state = readl(&xhci->op_regs->status);  in xhci_reset()
179 		xhci_warn(xhci, "Host not accessible, reset failed.\n");  in xhci_reset()
180 		return -ENODEV;  in xhci_reset()
184 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");  in xhci_reset()
188 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");  in xhci_reset()
189 	command = readl(&xhci->op_regs->command);  in xhci_reset()
191 	writel(command, &xhci->op_regs->command);  in xhci_reset()
193 	/* Existing Intel xHCI controllers require a delay of 1 mS,  in xhci_reset()
200 	if (xhci->quirks & XHCI_INTEL_HOST)  in xhci_reset()
203 	ret = xhci_handshake(&xhci->op_regs->command, CMD_RESET, 0, timeout_us);  in xhci_reset()
207 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)  in xhci_reset()
208 		usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));  in xhci_reset()
210 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_reset()
213 	 * xHCI cannot write to any doorbells or operational registers other  in xhci_reset()
216 	ret = xhci_handshake(&xhci->op_regs->status, STS_CNR, 0, timeout_us);  in xhci_reset()
218 	xhci->usb2_rhub.bus_state.port_c_suspend = 0;  in xhci_reset()
219 	xhci->usb2_rhub.bus_state.suspended_ports = 0;  in xhci_reset()
220 	xhci->usb2_rhub.bus_state.resuming_ports = 0;  in xhci_reset()
221 	xhci->usb3_rhub.bus_state.port_c_suspend = 0;  in xhci_reset()
222 	xhci->usb3_rhub.bus_state.suspended_ports = 0;  in xhci_reset()
223 	xhci->usb3_rhub.bus_state.resuming_ports = 0;  in xhci_reset()
228 static void xhci_zero_64b_regs(struct xhci_hcd *xhci)  in xhci_zero_64b_regs()  argument
230 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;  in xhci_zero_64b_regs()
249 	if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !device_iommu_mapped(dev))  in xhci_zero_64b_regs()
252 	xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");  in xhci_zero_64b_regs()
255 	val = readl(&xhci->op_regs->command);  in xhci_zero_64b_regs()
257 	writel(val, &xhci->op_regs->command);  in xhci_zero_64b_regs()
260 	val = readl(&xhci->op_regs->status);  in xhci_zero_64b_regs()
262 	writel(val, &xhci->op_regs->status);  in xhci_zero_64b_regs()
265 	val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);  in xhci_zero_64b_regs()
267 		xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);  in xhci_zero_64b_regs()
268 	val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);  in xhci_zero_64b_regs()
270 		xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);  in xhci_zero_64b_regs()
272 	intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),  in xhci_zero_64b_regs()
273 		      ARRAY_SIZE(xhci->run_regs->ir_set));  in xhci_zero_64b_regs()
278 		ir = &xhci->run_regs->ir_set[i];  in xhci_zero_64b_regs()
279 		val = xhci_read_64(xhci, &ir->erst_base);  in xhci_zero_64b_regs()
281 			xhci_write_64(xhci, 0, &ir->erst_base);  in xhci_zero_64b_regs()
282 		val= xhci_read_64(xhci, &ir->erst_dequeue);  in xhci_zero_64b_regs()
284 			xhci_write_64(xhci, 0, &ir->erst_dequeue);  in xhci_zero_64b_regs()
288 	err = xhci_handshake(&xhci->op_regs->status,  in xhci_zero_64b_regs()
292 		xhci_info(xhci, "Fault detected\n");  in xhci_zero_64b_regs()
299 static int xhci_setup_msi(struct xhci_hcd *xhci)  in xhci_setup_msi()  argument
305 	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);  in xhci_setup_msi()
309 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_setup_msi()
314 	ret = request_irq(pdev->irq, xhci_msi_irq,  in xhci_setup_msi()
315 				0, "xhci_hcd", xhci_to_hcd(xhci));  in xhci_setup_msi()
317 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_setup_msi()
326  * Set up MSI-X
328 static int xhci_setup_msix(struct xhci_hcd *xhci)  in xhci_setup_msix()  argument
331 	struct usb_hcd *hcd = xhci_to_hcd(xhci);  in xhci_setup_msix()
332 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);  in xhci_setup_msix()
335 	 * calculate number of msi-x vectors supported.  in xhci_setup_msix()
336 	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,  in xhci_setup_msix()
337 	 *   with max number of interrupters based on the xhci HCSPARAMS1.  in xhci_setup_msix()
338 	 * - num_online_cpus: maximum msi-x vectors per CPUs core.  in xhci_setup_msix()
341 	xhci->msix_count = min(num_online_cpus() + 1,  in xhci_setup_msix()
342 				HCS_MAX_INTRS(xhci->hcs_params1));  in xhci_setup_msix()
344 	ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,  in xhci_setup_msix()
347 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_setup_msix()
348 				"Failed to enable MSI-X");  in xhci_setup_msix()
352 	for (i = 0; i < xhci->msix_count; i++) {  in xhci_setup_msix()
354 				"xhci_hcd", xhci_to_hcd(xhci));  in xhci_setup_msix()
359 	hcd->msix_enabled = 1;  in xhci_setup_msix()
363 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");  in xhci_setup_msix()
364 	while (--i >= 0)  in xhci_setup_msix()
365 		free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));  in xhci_setup_msix()
370 /* Free any IRQs and disable MSI-X */
371 static void xhci_cleanup_msix(struct xhci_hcd *xhci)  in xhci_cleanup_msix()  argument
373 	struct usb_hcd *hcd = xhci_to_hcd(xhci);  in xhci_cleanup_msix()
374 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);  in xhci_cleanup_msix()
376 	if (xhci->quirks & XHCI_PLAT)  in xhci_cleanup_msix()
380 	if (hcd->irq > 0)  in xhci_cleanup_msix()
383 	if (hcd->msix_enabled) {  in xhci_cleanup_msix()
386 		for (i = 0; i < xhci->msix_count; i++)  in xhci_cleanup_msix()
387 			free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));  in xhci_cleanup_msix()
389 		free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));  in xhci_cleanup_msix()
393 	hcd->msix_enabled = 0;  in xhci_cleanup_msix()
396 static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)  in xhci_msix_sync_irqs()  argument
398 	struct usb_hcd *hcd = xhci_to_hcd(xhci);  in xhci_msix_sync_irqs()
400 	if (hcd->msix_enabled) {  in xhci_msix_sync_irqs()
401 		struct pci_dev *pdev = to_pci_dev(hcd->self.controller);  in xhci_msix_sync_irqs()
404 		for (i = 0; i < xhci->msix_count; i++)  in xhci_msix_sync_irqs()
411 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_try_enable_msi()  local
415 	/* The xhci platform device has set up IRQs through usb_add_hcd. */  in xhci_try_enable_msi()
416 	if (xhci->quirks & XHCI_PLAT)  in xhci_try_enable_msi()
419 	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);  in xhci_try_enable_msi()
424 	if (xhci->quirks & XHCI_BROKEN_MSI)  in xhci_try_enable_msi()
428 	if (hcd->irq)  in xhci_try_enable_msi()
429 		free_irq(hcd->irq, hcd);  in xhci_try_enable_msi()
430 	hcd->irq = 0;  in xhci_try_enable_msi()
432 	ret = xhci_setup_msix(xhci);  in xhci_try_enable_msi()
435 		ret = xhci_setup_msi(xhci);  in xhci_try_enable_msi()
438 		hcd->msi_enabled = 1;  in xhci_try_enable_msi()
442 	if (!pdev->irq) {  in xhci_try_enable_msi()
443 		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");  in xhci_try_enable_msi()
444 		return -EINVAL;  in xhci_try_enable_msi()
448 	if (!strlen(hcd->irq_descr))  in xhci_try_enable_msi()
449 		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",  in xhci_try_enable_msi()
450 			 hcd->driver->description, hcd->self.busnum);  in xhci_try_enable_msi()
453 	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,  in xhci_try_enable_msi()
454 			hcd->irq_descr, hcd);  in xhci_try_enable_msi()
456 		xhci_err(xhci, "request interrupt %d failed\n",  in xhci_try_enable_msi()
457 				pdev->irq);  in xhci_try_enable_msi()
460 	hcd->irq = pdev->irq;  in xhci_try_enable_msi()
471 static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)  in xhci_cleanup_msix()  argument
475 static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)  in xhci_msix_sync_irqs()  argument
483 	struct xhci_hcd *xhci;  in compliance_mode_recovery()  local
489 	xhci = from_timer(xhci, t, comp_mode_recovery_timer);  in compliance_mode_recovery()
490 	rhub = &xhci->usb3_rhub;  in compliance_mode_recovery()
491 	hcd = rhub->hcd;  in compliance_mode_recovery()
496 	for (i = 0; i < rhub->num_ports; i++) {  in compliance_mode_recovery()
497 		temp = readl(rhub->ports[i]->addr);  in compliance_mode_recovery()
503 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in compliance_mode_recovery()
504 					"Compliance mode detected->port %d",  in compliance_mode_recovery()
506 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in compliance_mode_recovery()
509 			if (hcd->state == HC_STATE_SUSPENDED)  in compliance_mode_recovery()
516 	if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))  in compliance_mode_recovery()
517 		mod_timer(&xhci->comp_mode_recovery_timer,  in compliance_mode_recovery()
522  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
528  * status event is generated when entering compliance mode (per xhci spec),
531 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)  in compliance_mode_recovery_timer_init()  argument
533 	xhci->port_status_u0 = 0;  in compliance_mode_recovery_timer_init()
534 	timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,  in compliance_mode_recovery_timer_init()
536 	xhci->comp_mode_recovery_timer.expires = jiffies +  in compliance_mode_recovery_timer_init()
539 	add_timer(&xhci->comp_mode_recovery_timer);  in compliance_mode_recovery_timer_init()
540 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in compliance_mode_recovery_timer_init()
546  * USB3.0 re-driver and that need the Compliance Mode Quirk.
548  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
559 	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))  in xhci_compliance_mode_recovery_timer_quirk_check()
571 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)  in xhci_all_ports_seen_u0()  argument
573 	return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));  in xhci_all_ports_seen_u0()
578  * Initialize memory for HCD and xHC (one-time init).
586 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_init()  local
589 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");  in xhci_init()
590 	spin_lock_init(&xhci->lock);  in xhci_init()
591 	if (xhci->hci_version == 0x95 && link_quirk) {  in xhci_init()
592 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_init()
594 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;  in xhci_init()
596 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_init()
597 				"xHCI doesn't need link TRB QUIRK");  in xhci_init()
599 	retval = xhci_mem_init(xhci, GFP_KERNEL);  in xhci_init()
600 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");  in xhci_init()
604 		xhci->quirks |= XHCI_COMP_MODE_QUIRK;  in xhci_init()
605 		compliance_mode_recovery_timer_init(xhci);  in xhci_init()
611 /*-------------------------------------------------------------------------*/
614 static int xhci_run_finished(struct xhci_hcd *xhci)  in xhci_run_finished()  argument
620 	 * Enable interrupts before starting the host (xhci 4.2 and 5.5.2).  in xhci_run_finished()
623 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_run_finished()
625 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable interrupts");  in xhci_run_finished()
626 	temp = readl(&xhci->op_regs->command);  in xhci_run_finished()
628 	writel(temp, &xhci->op_regs->command);  in xhci_run_finished()
630 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Enable primary interrupter");  in xhci_run_finished()
631 	temp = readl(&xhci->ir_set->irq_pending);  in xhci_run_finished()
632 	writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);  in xhci_run_finished()
634 	if (xhci_start(xhci)) {  in xhci_run_finished()
635 		xhci_halt(xhci);  in xhci_run_finished()
636 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_run_finished()
637 		return -ENODEV;  in xhci_run_finished()
640 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;  in xhci_run_finished()
642 	if (xhci->quirks & XHCI_NEC_HOST)  in xhci_run_finished()
643 		xhci_ring_cmd_db(xhci);  in xhci_run_finished()
645 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_run_finished()
660  * Setup MSI-X vectors and enable interrupts.
667 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_run()  local
669 	/* Start the xHCI host controller running only after the USB 2.0 roothub  in xhci_run()
673 	hcd->uses_new_polling = 1;  in xhci_run()
675 		return xhci_run_finished(xhci);  in xhci_run()
677 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");  in xhci_run()
683 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);  in xhci_run()
685 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_run()
688 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_run()
690 	temp = readl(&xhci->ir_set->irq_control);  in xhci_run()
692 	temp |= (xhci->imod_interval / 250) & ER_IRQ_INTERVAL_MASK;  in xhci_run()
693 	writel(temp, &xhci->ir_set->irq_control);  in xhci_run()
695 	if (xhci->quirks & XHCI_NEC_HOST) {  in xhci_run()
698 		command = xhci_alloc_command(xhci, false, GFP_KERNEL);  in xhci_run()
700 			return -ENOMEM;  in xhci_run()
702 		ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,  in xhci_run()
705 			xhci_free_command(xhci, command);  in xhci_run()
707 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_run()
710 	xhci_create_dbc_dev(xhci);  in xhci_run()
712 	xhci_debugfs_init(xhci);  in xhci_run()
714 	if (xhci_has_one_roothub(xhci))  in xhci_run()
715 		return xhci_run_finished(xhci);  in xhci_run()
717 	set_bit(HCD_FLAG_DEFER_RH_REGISTER, &hcd->flags);  in xhci_run()
724  * Stop xHCI driver.
735 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_stop()  local
737 	mutex_lock(&xhci->mutex);  in xhci_stop()
741 		mutex_unlock(&xhci->mutex);  in xhci_stop()
745 	xhci_remove_dbc_dev(xhci);  in xhci_stop()
747 	spin_lock_irq(&xhci->lock);  in xhci_stop()
748 	xhci->xhc_state |= XHCI_STATE_HALTED;  in xhci_stop()
749 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;  in xhci_stop()
750 	xhci_halt(xhci);  in xhci_stop()
751 	xhci_reset(xhci, XHCI_RESET_SHORT_USEC);  in xhci_stop()
752 	spin_unlock_irq(&xhci->lock);  in xhci_stop()
754 	xhci_cleanup_msix(xhci);  in xhci_stop()
757 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&  in xhci_stop()
758 			(!(xhci_all_ports_seen_u0(xhci)))) {  in xhci_stop()
759 		del_timer_sync(&xhci->comp_mode_recovery_timer);  in xhci_stop()
760 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_stop()
765 	if (xhci->quirks & XHCI_AMD_PLL_FIX)  in xhci_stop()
768 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_stop()
770 	temp = readl(&xhci->op_regs->status);  in xhci_stop()
771 	writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);  in xhci_stop()
772 	temp = readl(&xhci->ir_set->irq_pending);  in xhci_stop()
773 	writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);  in xhci_stop()
775 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");  in xhci_stop()
776 	xhci_mem_cleanup(xhci);  in xhci_stop()
777 	xhci_debugfs_exit(xhci);  in xhci_stop()
778 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_stop()
779 			"xhci_stop completed - status = %x",  in xhci_stop()
780 			readl(&xhci->op_regs->status));  in xhci_stop()
781 	mutex_unlock(&xhci->mutex);  in xhci_stop()
785  * Shutdown HC (not bus-specific)
795 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_shutdown()  local
797 	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)  in xhci_shutdown()
798 		usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));  in xhci_shutdown()
801 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",  in xhci_shutdown()
802 			__func__, hcd->self.busnum);  in xhci_shutdown()
803 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);  in xhci_shutdown()
804 	del_timer_sync(&hcd->rh_timer);  in xhci_shutdown()
806 	if (xhci->shared_hcd) {  in xhci_shutdown()
807 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);  in xhci_shutdown()
808 		del_timer_sync(&xhci->shared_hcd->rh_timer);  in xhci_shutdown()
811 	spin_lock_irq(&xhci->lock);  in xhci_shutdown()
812 	xhci_halt(xhci);  in xhci_shutdown()
816 	 * firmware delay in ADL-P PCH if port are left in U3 at shutdown  in xhci_shutdown()
818 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP ||  in xhci_shutdown()
819 	    xhci->quirks & XHCI_RESET_TO_DEFAULT)  in xhci_shutdown()
820 		xhci_reset(xhci, XHCI_RESET_SHORT_USEC);  in xhci_shutdown()
822 	spin_unlock_irq(&xhci->lock);  in xhci_shutdown()
824 	xhci_cleanup_msix(xhci);  in xhci_shutdown()
826 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_shutdown()
827 			"xhci_shutdown completed - status = %x",  in xhci_shutdown()
828 			readl(&xhci->op_regs->status));  in xhci_shutdown()
833 static void xhci_save_registers(struct xhci_hcd *xhci)  in xhci_save_registers()  argument
835 	xhci->s3.command = readl(&xhci->op_regs->command);  in xhci_save_registers()
836 	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);  in xhci_save_registers()
837 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);  in xhci_save_registers()
838 	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);  in xhci_save_registers()
839 	xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);  in xhci_save_registers()
840 	xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);  in xhci_save_registers()
841 	xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);  in xhci_save_registers()
842 	xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);  in xhci_save_registers()
843 	xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);  in xhci_save_registers()
846 static void xhci_restore_registers(struct xhci_hcd *xhci)  in xhci_restore_registers()  argument
848 	writel(xhci->s3.command, &xhci->op_regs->command);  in xhci_restore_registers()
849 	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);  in xhci_restore_registers()
850 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);  in xhci_restore_registers()
851 	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);  in xhci_restore_registers()
852 	writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);  in xhci_restore_registers()
853 	xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);  in xhci_restore_registers()
854 	xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);  in xhci_restore_registers()
855 	writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);  in xhci_restore_registers()
856 	writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);  in xhci_restore_registers()
859 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)  in xhci_set_cmd_ring_deq()  argument
864 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);  in xhci_set_cmd_ring_deq()
866 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,  in xhci_set_cmd_ring_deq()
867 				      xhci->cmd_ring->dequeue) &  in xhci_set_cmd_ring_deq()
869 		xhci->cmd_ring->cycle_state;  in xhci_set_cmd_ring_deq()
870 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,  in xhci_set_cmd_ring_deq()
873 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);  in xhci_set_cmd_ring_deq()
880  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
883  * middle of the ring (TRBs are 16-byte aligned).
885 static void xhci_clear_command_ring(struct xhci_hcd *xhci)  in xhci_clear_command_ring()  argument
890 	ring = xhci->cmd_ring;  in xhci_clear_command_ring()
891 	seg = ring->deq_seg;  in xhci_clear_command_ring()
893 		memset(seg->trbs, 0,  in xhci_clear_command_ring()
894 			sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));  in xhci_clear_command_ring()
895 		seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=  in xhci_clear_command_ring()
897 		seg = seg->next;  in xhci_clear_command_ring()
898 	} while (seg != ring->deq_seg);  in xhci_clear_command_ring()
901 	ring->deq_seg = ring->first_seg;  in xhci_clear_command_ring()
902 	ring->dequeue = ring->first_seg->trbs;  in xhci_clear_command_ring()
903 	ring->enq_seg = ring->deq_seg;  in xhci_clear_command_ring()
904 	ring->enqueue = ring->dequeue;  in xhci_clear_command_ring()
906 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;  in xhci_clear_command_ring()
911 	ring->cycle_state = 1;  in xhci_clear_command_ring()
915 	 * Yes, this will need to be re-written after resume, but we're paranoid  in xhci_clear_command_ring()
920 	xhci_set_cmd_ring_deq(xhci);  in xhci_clear_command_ring()
928  * Internal wake causes immediate xHCI wake after suspend. PORT_CSC write done
932 static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci,  in xhci_disable_hub_port_wake()  argument
940 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_disable_hub_port_wake()
942 	for (i = 0; i < rhub->num_ports; i++) {  in xhci_disable_hub_port_wake()
943 		portsc = readl(rhub->ports[i]->addr);  in xhci_disable_hub_port_wake()
956 			writel(t2, rhub->ports[i]->addr);  in xhci_disable_hub_port_wake()
957 			xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n",  in xhci_disable_hub_port_wake()
958 				 rhub->hcd->self.busnum, i + 1, portsc, t2);  in xhci_disable_hub_port_wake()
961 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_disable_hub_port_wake()
964 static bool xhci_pending_portevent(struct xhci_hcd *xhci)  in xhci_pending_portevent()  argument
971 	status = readl(&xhci->op_regs->status);  in xhci_pending_portevent()
977 	 * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.  in xhci_pending_portevent()
980 	port_index = xhci->usb2_rhub.num_ports;  in xhci_pending_portevent()
981 	ports = xhci->usb2_rhub.ports;  in xhci_pending_portevent()
982 	while (port_index--) {  in xhci_pending_portevent()
983 		portsc = readl(ports[port_index]->addr);  in xhci_pending_portevent()
988 	port_index = xhci->usb3_rhub.num_ports;  in xhci_pending_portevent()
989 	ports = xhci->usb3_rhub.ports;  in xhci_pending_portevent()
990 	while (port_index--) {  in xhci_pending_portevent()
991 		portsc = readl(ports[port_index]->addr);  in xhci_pending_portevent()
1000  * Stop HC (not bus-specific)
1005 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)  in xhci_suspend()  argument
1009 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);  in xhci_suspend()
1013 	if (!hcd->state)  in xhci_suspend()
1016 	if (hcd->state != HC_STATE_SUSPENDED ||  in xhci_suspend()
1017 	    (xhci->shared_hcd && xhci->shared_hcd->state != HC_STATE_SUSPENDED))  in xhci_suspend()
1018 		return -EINVAL;  in xhci_suspend()
1021 	xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup);  in xhci_suspend()
1022 	xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup);  in xhci_suspend()
1027 	xhci_dbc_suspend(xhci);  in xhci_suspend()
1030 	xhci_dbg(xhci, "%s: stopping usb%d port polling.\n",  in xhci_suspend()
1031 		 __func__, hcd->self.busnum);  in xhci_suspend()
1032 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);  in xhci_suspend()
1033 	del_timer_sync(&hcd->rh_timer);  in xhci_suspend()
1034 	if (xhci->shared_hcd) {  in xhci_suspend()
1035 		clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);  in xhci_suspend()
1036 		del_timer_sync(&xhci->shared_hcd->rh_timer);  in xhci_suspend()
1039 	if (xhci->quirks & XHCI_SUSPEND_DELAY)  in xhci_suspend()
1042 	spin_lock_irq(&xhci->lock);  in xhci_suspend()
1043 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);  in xhci_suspend()
1044 	if (xhci->shared_hcd)  in xhci_suspend()
1045 		clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);  in xhci_suspend()
1050 	command = readl(&xhci->op_regs->command);  in xhci_suspend()
1052 	writel(command, &xhci->op_regs->command);  in xhci_suspend()
1055 	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;  in xhci_suspend()
1057 	if (xhci_handshake(&xhci->op_regs->status,  in xhci_suspend()
1059 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");  in xhci_suspend()
1060 		spin_unlock_irq(&xhci->lock);  in xhci_suspend()
1061 		return -ETIMEDOUT;  in xhci_suspend()
1063 	xhci_clear_command_ring(xhci);  in xhci_suspend()
1066 	xhci_save_registers(xhci);  in xhci_suspend()
1069 	command = readl(&xhci->op_regs->command);  in xhci_suspend()
1071 	writel(command, &xhci->op_regs->command);  in xhci_suspend()
1072 	xhci->broken_suspend = 0;  in xhci_suspend()
1073 	if (xhci_handshake(&xhci->op_regs->status,  in xhci_suspend()
1081 	 * if SRE and HCE bits are not set (as per xhci  in xhci_suspend()
1084 		res = readl(&xhci->op_regs->status);  in xhci_suspend()
1085 		if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&  in xhci_suspend()
1088 			xhci->broken_suspend = 1;  in xhci_suspend()
1090 			xhci_warn(xhci, "WARN: xHC save state timeout\n");  in xhci_suspend()
1091 			spin_unlock_irq(&xhci->lock);  in xhci_suspend()
1092 			return -ETIMEDOUT;  in xhci_suspend()
1095 	spin_unlock_irq(&xhci->lock);  in xhci_suspend()
1098 	 * Deleting Compliance Mode Recovery Timer because the xHCI Host  in xhci_suspend()
1101 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&  in xhci_suspend()
1102 			(!(xhci_all_ports_seen_u0(xhci)))) {  in xhci_suspend()
1103 		del_timer_sync(&xhci->comp_mode_recovery_timer);  in xhci_suspend()
1104 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_suspend()
1110 	/* synchronize irq when using MSI-X */  in xhci_suspend()
1111 	xhci_msix_sync_irqs(xhci);  in xhci_suspend()
1118  * start xHC (not bus-specific)
1123 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)  in xhci_resume()  argument
1126 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);  in xhci_resume()
1132 	if (!hcd->state)  in xhci_resume()
1139 	if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||  in xhci_resume()
1140 	    time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))  in xhci_resume()
1143 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);  in xhci_resume()
1144 	if (xhci->shared_hcd)  in xhci_resume()
1145 		set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);  in xhci_resume()
1147 	spin_lock_irq(&xhci->lock);  in xhci_resume()
1149 	if (hibernated || xhci->quirks & XHCI_RESET_ON_RESUME || xhci->broken_suspend)  in xhci_resume()
1157 		retval = xhci_handshake(&xhci->op_regs->status,  in xhci_resume()
1160 			xhci_warn(xhci, "Controller not ready at resume %d\n",  in xhci_resume()
1162 			spin_unlock_irq(&xhci->lock);  in xhci_resume()
1166 		xhci_restore_registers(xhci);  in xhci_resume()
1168 		xhci_set_cmd_ring_deq(xhci);  in xhci_resume()
1171 		command = readl(&xhci->op_regs->command);  in xhci_resume()
1173 		writel(command, &xhci->op_regs->command);  in xhci_resume()
1176 		 * restore so setting the timeout to 100ms. Xhci specification  in xhci_resume()
1179 		if (xhci_handshake(&xhci->op_regs->status,  in xhci_resume()
1181 			xhci_warn(xhci, "WARN: xHC restore state timeout\n");  in xhci_resume()
1182 			spin_unlock_irq(&xhci->lock);  in xhci_resume()
1183 			return -ETIMEDOUT;  in xhci_resume()
1187 	temp = readl(&xhci->op_regs->status);  in xhci_resume()
1189 	/* re-initialize the HC on Restore Error, or Host Controller Error */  in xhci_resume()
1192 		if (!xhci->broken_suspend)  in xhci_resume()
1193 			xhci_warn(xhci, "xHC error in resume, USBSTS 0x%x, Reinit\n", temp);  in xhci_resume()
1197 		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&  in xhci_resume()
1198 				!(xhci_all_ports_seen_u0(xhci))) {  in xhci_resume()
1199 			del_timer_sync(&xhci->comp_mode_recovery_timer);  in xhci_resume()
1200 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_resume()
1205 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);  in xhci_resume()
1206 		if (xhci->shared_hcd)  in xhci_resume()
1207 			usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);  in xhci_resume()
1209 		xhci_dbg(xhci, "Stop HCD\n");  in xhci_resume()
1210 		xhci_halt(xhci);  in xhci_resume()
1211 		xhci_zero_64b_regs(xhci);  in xhci_resume()
1212 		retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);  in xhci_resume()
1213 		spin_unlock_irq(&xhci->lock);  in xhci_resume()
1216 		xhci_cleanup_msix(xhci);  in xhci_resume()
1218 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");  in xhci_resume()
1219 		temp = readl(&xhci->op_regs->status);  in xhci_resume()
1220 		writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);  in xhci_resume()
1221 		temp = readl(&xhci->ir_set->irq_pending);  in xhci_resume()
1222 		writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);  in xhci_resume()
1224 		xhci_dbg(xhci, "cleaning up memory\n");  in xhci_resume()
1225 		xhci_mem_cleanup(xhci);  in xhci_resume()
1226 		xhci_debugfs_exit(xhci);  in xhci_resume()
1227 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",  in xhci_resume()
1228 			    readl(&xhci->op_regs->status));  in xhci_resume()
1234 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");  in xhci_resume()
1240 		xhci_dbg(xhci, "Start the primary HCD\n");  in xhci_resume()
1242 		if (!retval && xhci->shared_hcd) {  in xhci_resume()
1243 			xhci_dbg(xhci, "Start the secondary HCD\n");  in xhci_resume()
1244 			retval = xhci_run(xhci->shared_hcd);  in xhci_resume()
1247 		hcd->state = HC_STATE_SUSPENDED;  in xhci_resume()
1248 		if (xhci->shared_hcd)  in xhci_resume()
1249 			xhci->shared_hcd->state = HC_STATE_SUSPENDED;  in xhci_resume()
1254 	command = readl(&xhci->op_regs->command);  in xhci_resume()
1256 	writel(command, &xhci->op_regs->command);  in xhci_resume()
1257 	xhci_handshake(&xhci->op_regs->status, STS_HALT,  in xhci_resume()
1269 	spin_unlock_irq(&xhci->lock);  in xhci_resume()
1271 	xhci_dbc_resume(xhci);  in xhci_resume()
1280 		pending_portevent = xhci_pending_portevent(xhci);  in xhci_resume()
1283 			pending_portevent = xhci_pending_portevent(xhci);  in xhci_resume()
1287 			if (xhci->shared_hcd)  in xhci_resume()
1288 				usb_hcd_resume_root_hub(xhci->shared_hcd);  in xhci_resume()
1294 	 * be re-initialized Always after a system resume. Ports are subject  in xhci_resume()
1298 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)  in xhci_resume()
1299 		compliance_mode_recovery_timer_init(xhci);  in xhci_resume()
1301 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)  in xhci_resume()
1302 		usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));  in xhci_resume()
1304 	/* Re-enable port polling. */  in xhci_resume()
1305 	xhci_dbg(xhci, "%s: starting usb%d port polling.\n",  in xhci_resume()
1306 		 __func__, hcd->self.busnum);  in xhci_resume()
1307 	if (xhci->shared_hcd) {  in xhci_resume()
1308 		set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);  in xhci_resume()
1309 		usb_hcd_poll_rh_status(xhci->shared_hcd);  in xhci_resume()
1311 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);  in xhci_resume()
1319 /*-------------------------------------------------------------------------*/
1329 	buf_len = urb->transfer_buffer_length;  in xhci_map_temp_buffer()
1332 			    dev_to_node(hcd->self.sysdev));  in xhci_map_temp_buffer()
1335 		sg_pcopy_to_buffer(urb->sg, urb->num_sgs,  in xhci_map_temp_buffer()
1338 	urb->transfer_buffer = temp;  in xhci_map_temp_buffer()
1339 	urb->transfer_dma = dma_map_single(hcd->self.sysdev,  in xhci_map_temp_buffer()
1340 					   urb->transfer_buffer,  in xhci_map_temp_buffer()
1341 					   urb->transfer_buffer_length,  in xhci_map_temp_buffer()
1344 	if (dma_mapping_error(hcd->self.sysdev,  in xhci_map_temp_buffer()
1345 			      urb->transfer_dma)) {  in xhci_map_temp_buffer()
1346 		ret = -EAGAIN;  in xhci_map_temp_buffer()
1349 		urb->transfer_flags |= URB_DMA_MAP_SINGLE;  in xhci_map_temp_buffer()
1366 	tail_sg = urb->sg;  in xhci_urb_temp_buffer_required()
1367 	max_pkt = usb_endpoint_maxp(&urb->ep->desc);  in xhci_urb_temp_buffer_required()
1369 	if (!urb->num_sgs)  in xhci_urb_temp_buffer_required()
1372 	if (urb->dev->speed >= USB_SPEED_SUPER)  in xhci_urb_temp_buffer_required()
1377 	if (urb->transfer_buffer_length != 0 &&  in xhci_urb_temp_buffer_required()
1378 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) {  in xhci_urb_temp_buffer_required()
1379 		for_each_sg(urb->sg, sg, urb->num_sgs, i) {  in xhci_urb_temp_buffer_required()
1380 			len = len + sg->length;  in xhci_urb_temp_buffer_required()
1381 			if (i > trb_size - 2) {  in xhci_urb_temp_buffer_required()
1382 				len = len - tail_sg->length;  in xhci_urb_temp_buffer_required()
1403 	buf_len = urb->transfer_buffer_length;  in xhci_unmap_temp_buf()
1406 	    (urb->transfer_flags & URB_DMA_MAP_SINGLE))  in xhci_unmap_temp_buf()
1407 		dma_unmap_single(hcd->self.sysdev,  in xhci_unmap_temp_buf()
1408 				 urb->transfer_dma,  in xhci_unmap_temp_buf()
1409 				 urb->transfer_buffer_length,  in xhci_unmap_temp_buf()
1413 		len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs,  in xhci_unmap_temp_buf()
1414 					   urb->transfer_buffer,  in xhci_unmap_temp_buf()
1420 			urb->actual_length = len;  in xhci_unmap_temp_buf()
1423 	urb->transfer_flags &= ~URB_DMA_MAP_SINGLE;  in xhci_unmap_temp_buf()
1424 	kfree(urb->transfer_buffer);  in xhci_unmap_temp_buf()
1425 	urb->transfer_buffer = NULL;  in xhci_unmap_temp_buf()
1437 	struct xhci_hcd *xhci;  in xhci_map_urb_for_dma()  local
1439 	xhci = hcd_to_xhci(hcd);  in xhci_map_urb_for_dma()
1444 	if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) {  in xhci_map_urb_for_dma()
1453 	struct xhci_hcd *xhci;  in xhci_unmap_urb_for_dma()  local
1456 	xhci = hcd_to_xhci(hcd);  in xhci_unmap_urb_for_dma()
1458 	if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE))  in xhci_unmap_urb_for_dma()
1461 	if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf)  in xhci_unmap_urb_for_dma()
1468  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1472  * Index  = (epnum * 2) + direction - 1,
1475  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1484 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;  in xhci_get_endpoint_index()
1490  * address from the XHCI endpoint index.
1516 	return fls(added_ctxs) - 1;  in xhci_last_valid_endpoint()
1520  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1525 	struct xhci_hcd	*xhci;  in xhci_check_args()  local
1529 		pr_debug("xHCI %s called with invalid args\n", func);  in xhci_check_args()
1530 		return -EINVAL;  in xhci_check_args()
1532 	if (!udev->parent) {  in xhci_check_args()
1533 		pr_debug("xHCI %s called for root hub\n", func);  in xhci_check_args()
1537 	xhci = hcd_to_xhci(hcd);  in xhci_check_args()
1539 		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {  in xhci_check_args()
1540 			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",  in xhci_check_args()
1542 			return -EINVAL;  in xhci_check_args()
1545 		virt_dev = xhci->devs[udev->slot_id];  in xhci_check_args()
1546 		if (virt_dev->udev != udev) {  in xhci_check_args()
1547 			xhci_dbg(xhci, "xHCI %s called with udev and "  in xhci_check_args()
1549 			return -EINVAL;  in xhci_check_args()
1553 	if (xhci->xhc_state & XHCI_STATE_HALTED)  in xhci_check_args()
1554 		return -ENODEV;  in xhci_check_args()
1559 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1569 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,  in xhci_check_maxpacket()  argument
1580 	out_ctx = xhci->devs[slot_id]->out_ctx;  in xhci_check_maxpacket()
1581 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);  in xhci_check_maxpacket()
1582 	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));  in xhci_check_maxpacket()
1583 	max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);  in xhci_check_maxpacket()
1585 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,  in xhci_check_maxpacket()
1587 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,  in xhci_check_maxpacket()
1590 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,  in xhci_check_maxpacket()
1591 				"Max packet size in xHCI HW = %d",  in xhci_check_maxpacket()
1593 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,  in xhci_check_maxpacket()
1597 		/* FIXME: This won't work if a non-default control endpoint  in xhci_check_maxpacket()
1601 		command = xhci_alloc_command(xhci, true, mem_flags);  in xhci_check_maxpacket()
1603 			return -ENOMEM;  in xhci_check_maxpacket()
1605 		command->in_ctx = xhci->devs[slot_id]->in_ctx;  in xhci_check_maxpacket()
1606 		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);  in xhci_check_maxpacket()
1608 			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_check_maxpacket()
1610 			ret = -ENOMEM;  in xhci_check_maxpacket()
1614 		xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,  in xhci_check_maxpacket()
1615 				xhci->devs[slot_id]->out_ctx, ep_index);  in xhci_check_maxpacket()
1617 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);  in xhci_check_maxpacket()
1618 		ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */  in xhci_check_maxpacket()
1619 		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);  in xhci_check_maxpacket()
1620 		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));  in xhci_check_maxpacket()
1622 		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);  in xhci_check_maxpacket()
1623 		ctrl_ctx->drop_flags = 0;  in xhci_check_maxpacket()
1625 		ret = xhci_configure_endpoint(xhci, urb->dev, command,  in xhci_check_maxpacket()
1631 		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);  in xhci_check_maxpacket()
1633 		kfree(command->completion);  in xhci_check_maxpacket()
1640  * non-error returns are a promise to giveback() the urb later
1645 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_urb_enqueue()  local
1654 		return -EINVAL;  in xhci_urb_enqueue()
1655 	ret = xhci_check_args(hcd, urb->dev, urb->ep,  in xhci_urb_enqueue()
1658 		return ret ? ret : -EINVAL;  in xhci_urb_enqueue()
1660 	slot_id = urb->dev->slot_id;  in xhci_urb_enqueue()
1661 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);  in xhci_urb_enqueue()
1662 	ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;  in xhci_urb_enqueue()
1665 		return -ESHUTDOWN;  in xhci_urb_enqueue()
1667 	if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {  in xhci_urb_enqueue()
1668 		xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");  in xhci_urb_enqueue()
1669 		return -ENODEV;  in xhci_urb_enqueue()
1672 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))  in xhci_urb_enqueue()
1673 		num_tds = urb->number_of_packets;  in xhci_urb_enqueue()
1674 	else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&  in xhci_urb_enqueue()
1675 	    urb->transfer_buffer_length > 0 &&  in xhci_urb_enqueue()
1676 	    urb->transfer_flags & URB_ZERO_PACKET &&  in xhci_urb_enqueue()
1677 	    !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))  in xhci_urb_enqueue()
1684 		return -ENOMEM;  in xhci_urb_enqueue()
1686 	urb_priv->num_tds = num_tds;  in xhci_urb_enqueue()
1687 	urb_priv->num_tds_done = 0;  in xhci_urb_enqueue()
1688 	urb->hcpriv = urb_priv;  in xhci_urb_enqueue()
1692 	if (usb_endpoint_xfer_control(&urb->ep->desc)) {  in xhci_urb_enqueue()
1696 		if (urb->dev->speed == USB_SPEED_FULL) {  in xhci_urb_enqueue()
1697 			ret = xhci_check_maxpacket(xhci, slot_id,  in xhci_urb_enqueue()
1701 				urb->hcpriv = NULL;  in xhci_urb_enqueue()
1707 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_urb_enqueue()
1709 	if (xhci->xhc_state & XHCI_STATE_DYING) {  in xhci_urb_enqueue()
1710 		xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",  in xhci_urb_enqueue()
1711 			 urb->ep->desc.bEndpointAddress, urb);  in xhci_urb_enqueue()
1712 		ret = -ESHUTDOWN;  in xhci_urb_enqueue()
1716 		xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",  in xhci_urb_enqueue()
1718 		ret = -EINVAL;  in xhci_urb_enqueue()
1722 		xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");  in xhci_urb_enqueue()
1723 		ret = -EINVAL;  in xhci_urb_enqueue()
1727 	switch (usb_endpoint_type(&urb->ep->desc)) {  in xhci_urb_enqueue()
1730 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,  in xhci_urb_enqueue()
1734 		ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,  in xhci_urb_enqueue()
1738 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,  in xhci_urb_enqueue()
1742 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,  in xhci_urb_enqueue()
1749 		urb->hcpriv = NULL;  in xhci_urb_enqueue()
1751 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_urb_enqueue()
1771  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1773  *     endpoint command, as noted in the xHCI 0.95 errata.
1791 	struct xhci_hcd *xhci;  in xhci_urb_dequeue()  local
1800 	xhci = hcd_to_xhci(hcd);  in xhci_urb_dequeue()
1801 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_urb_dequeue()
1811 	vdev = xhci->devs[urb->dev->slot_id];  in xhci_urb_dequeue()
1812 	urb_priv = urb->hcpriv;  in xhci_urb_dequeue()
1816 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);  in xhci_urb_dequeue()
1817 	ep = &vdev->eps[ep_index];  in xhci_urb_dequeue()
1818 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);  in xhci_urb_dequeue()
1823 	temp = readl(&xhci->op_regs->status);  in xhci_urb_dequeue()
1824 	if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {  in xhci_urb_dequeue()
1825 		xhci_hc_died(xhci);  in xhci_urb_dequeue()
1830 	 * check ring is not re-allocated since URB was enqueued. If it is, then  in xhci_urb_dequeue()
1834 	if (!td_on_ring(&urb_priv->td[0], ep_ring)) {  in xhci_urb_dequeue()
1835 		xhci_err(xhci, "Canceled URB td not found on endpoint ring");  in xhci_urb_dequeue()
1836 		for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {  in xhci_urb_dequeue()
1837 			td = &urb_priv->td[i];  in xhci_urb_dequeue()
1838 			if (!list_empty(&td->cancelled_td_list))  in xhci_urb_dequeue()
1839 				list_del_init(&td->cancelled_td_list);  in xhci_urb_dequeue()
1844 	if (xhci->xhc_state & XHCI_STATE_HALTED) {  in xhci_urb_dequeue()
1845 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,  in xhci_urb_dequeue()
1847 		for (i = urb_priv->num_tds_done;  in xhci_urb_dequeue()
1848 		     i < urb_priv->num_tds;  in xhci_urb_dequeue()
1850 			td = &urb_priv->td[i];  in xhci_urb_dequeue()
1851 			if (!list_empty(&td->td_list))  in xhci_urb_dequeue()
1852 				list_del_init(&td->td_list);  in xhci_urb_dequeue()
1853 			if (!list_empty(&td->cancelled_td_list))  in xhci_urb_dequeue()
1854 				list_del_init(&td->cancelled_td_list);  in xhci_urb_dequeue()
1859 	i = urb_priv->num_tds_done;  in xhci_urb_dequeue()
1860 	if (i < urb_priv->num_tds)  in xhci_urb_dequeue()
1861 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,  in xhci_urb_dequeue()
1864 				urb, urb->dev->devpath,  in xhci_urb_dequeue()
1865 				urb->ep->desc.bEndpointAddress,  in xhci_urb_dequeue()
1867 					urb_priv->td[i].start_seg,  in xhci_urb_dequeue()
1868 					urb_priv->td[i].first_trb));  in xhci_urb_dequeue()
1870 	for (; i < urb_priv->num_tds; i++) {  in xhci_urb_dequeue()
1871 		td = &urb_priv->td[i];  in xhci_urb_dequeue()
1873 		if (list_empty(&td->cancelled_td_list)) {  in xhci_urb_dequeue()
1874 			td->cancel_status = TD_DIRTY;  in xhci_urb_dequeue()
1875 			list_add_tail(&td->cancelled_td_list,  in xhci_urb_dequeue()
1876 				      &ep->cancelled_td_list);  in xhci_urb_dequeue()
1883 	if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {  in xhci_urb_dequeue()
1884 		command = xhci_alloc_command(xhci, false, GFP_ATOMIC);  in xhci_urb_dequeue()
1886 			ret = -ENOMEM;  in xhci_urb_dequeue()
1889 		ep->ep_state |= EP_STOP_CMD_PENDING;  in xhci_urb_dequeue()
1890 		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,  in xhci_urb_dequeue()
1892 		xhci_ring_cmd_db(xhci);  in xhci_urb_dequeue()
1895 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_urb_dequeue()
1902 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_urb_dequeue()
1903 	usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);  in xhci_urb_dequeue()
1918  * the xhci->devs[slot_id] structure.
1923 	struct xhci_hcd *xhci;  in xhci_drop_endpoint()  local
1935 	xhci = hcd_to_xhci(hcd);  in xhci_drop_endpoint()
1936 	if (xhci->xhc_state & XHCI_STATE_DYING)  in xhci_drop_endpoint()
1937 		return -ENODEV;  in xhci_drop_endpoint()
1939 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);  in xhci_drop_endpoint()
1940 	drop_flag = xhci_get_endpoint_flag(&ep->desc);  in xhci_drop_endpoint()
1942 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",  in xhci_drop_endpoint()
1947 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;  in xhci_drop_endpoint()
1948 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;  in xhci_drop_endpoint()
1951 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_drop_endpoint()
1956 	ep_index = xhci_get_endpoint_index(&ep->desc);  in xhci_drop_endpoint()
1957 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);  in xhci_drop_endpoint()
1962 	    le32_to_cpu(ctrl_ctx->drop_flags) &  in xhci_drop_endpoint()
1963 	    xhci_get_endpoint_flag(&ep->desc)) {  in xhci_drop_endpoint()
1965 		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)  in xhci_drop_endpoint()
1966 			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",  in xhci_drop_endpoint()
1971 	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);  in xhci_drop_endpoint()
1972 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);  in xhci_drop_endpoint()
1974 	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);  in xhci_drop_endpoint()
1975 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);  in xhci_drop_endpoint()
1977 	xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);  in xhci_drop_endpoint()
1979 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);  in xhci_drop_endpoint()
1981 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",  in xhci_drop_endpoint()
1982 			(unsigned int) ep->desc.bEndpointAddress,  in xhci_drop_endpoint()
1983 			udev->slot_id,  in xhci_drop_endpoint()
2001  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
2006 	struct xhci_hcd *xhci;  in xhci_add_endpoint()  local
2019 		ep->hcpriv = NULL;  in xhci_add_endpoint()
2022 	xhci = hcd_to_xhci(hcd);  in xhci_add_endpoint()
2023 	if (xhci->xhc_state & XHCI_STATE_DYING)  in xhci_add_endpoint()
2024 		return -ENODEV;  in xhci_add_endpoint()
2026 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);  in xhci_add_endpoint()
2032 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",  in xhci_add_endpoint()
2037 	virt_dev = xhci->devs[udev->slot_id];  in xhci_add_endpoint()
2038 	in_ctx = virt_dev->in_ctx;  in xhci_add_endpoint()
2041 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_add_endpoint()
2046 	ep_index = xhci_get_endpoint_index(&ep->desc);  in xhci_add_endpoint()
2050 	if (virt_dev->eps[ep_index].ring &&  in xhci_add_endpoint()
2051 			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {  in xhci_add_endpoint()
2052 		xhci_warn(xhci, "Trying to add endpoint 0x%x "  in xhci_add_endpoint()
2054 				(unsigned int) ep->desc.bEndpointAddress);  in xhci_add_endpoint()
2055 		return -EINVAL;  in xhci_add_endpoint()
2061 	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {  in xhci_add_endpoint()
2062 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",  in xhci_add_endpoint()
2072 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {  in xhci_add_endpoint()
2073 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",  in xhci_add_endpoint()
2074 				__func__, ep->desc.bEndpointAddress);  in xhci_add_endpoint()
2075 		return -ENOMEM;  in xhci_add_endpoint()
2078 	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);  in xhci_add_endpoint()
2079 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);  in xhci_add_endpoint()
2083 	 * this re-adds a new state for the endpoint from the new endpoint  in xhci_add_endpoint()
2084 	 * descriptors.  We must drop and re-add this endpoint, so we leave the  in xhci_add_endpoint()
2087 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);  in xhci_add_endpoint()
2090 	ep->hcpriv = udev;  in xhci_add_endpoint()
2092 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);  in xhci_add_endpoint()
2095 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",  in xhci_add_endpoint()
2096 			(unsigned int) ep->desc.bEndpointAddress,  in xhci_add_endpoint()
2097 			udev->slot_id,  in xhci_add_endpoint()
2104 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)  in xhci_zero_in_ctx()  argument
2111 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);  in xhci_zero_in_ctx()
2113 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_zero_in_ctx()
2123 	ctrl_ctx->drop_flags = 0;  in xhci_zero_in_ctx()
2124 	ctrl_ctx->add_flags = 0;  in xhci_zero_in_ctx()
2125 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);  in xhci_zero_in_ctx()
2126 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);  in xhci_zero_in_ctx()
2128 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));  in xhci_zero_in_ctx()
2130 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);  in xhci_zero_in_ctx()
2131 		ep_ctx->ep_info = 0;  in xhci_zero_in_ctx()
2132 		ep_ctx->ep_info2 = 0;  in xhci_zero_in_ctx()
2133 		ep_ctx->deq = 0;  in xhci_zero_in_ctx()
2134 		ep_ctx->tx_info = 0;  in xhci_zero_in_ctx()
2138 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,  in xhci_configure_endpoint_result()  argument
2146 		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");  in xhci_configure_endpoint_result()
2147 		ret = -ETIME;  in xhci_configure_endpoint_result()
2150 		dev_warn(&udev->dev,  in xhci_configure_endpoint_result()
2152 		ret = -ENOMEM;  in xhci_configure_endpoint_result()
2157 		dev_warn(&udev->dev,  in xhci_configure_endpoint_result()
2159 		ret = -ENOSPC;  in xhci_configure_endpoint_result()
2164 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "  in xhci_configure_endpoint_result()
2167 		ret = -EINVAL;  in xhci_configure_endpoint_result()
2170 		dev_warn(&udev->dev,  in xhci_configure_endpoint_result()
2172 		ret = -ENODEV;  in xhci_configure_endpoint_result()
2175 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,  in xhci_configure_endpoint_result()
2180 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",  in xhci_configure_endpoint_result()
2182 		ret = -EINVAL;  in xhci_configure_endpoint_result()
2188 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,  in xhci_evaluate_context_result()  argument
2196 		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");  in xhci_evaluate_context_result()
2197 		ret = -ETIME;  in xhci_evaluate_context_result()
2200 		dev_warn(&udev->dev,  in xhci_evaluate_context_result()
2201 			 "WARN: xHCI driver setup invalid evaluate context command.\n");  in xhci_evaluate_context_result()
2202 		ret = -EINVAL;  in xhci_evaluate_context_result()
2205 		dev_warn(&udev->dev,  in xhci_evaluate_context_result()
2207 		ret = -EINVAL;  in xhci_evaluate_context_result()
2210 		dev_warn(&udev->dev,  in xhci_evaluate_context_result()
2212 		ret = -EINVAL;  in xhci_evaluate_context_result()
2215 		dev_warn(&udev->dev,  in xhci_evaluate_context_result()
2217 		ret = -ENODEV;  in xhci_evaluate_context_result()
2221 		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");  in xhci_evaluate_context_result()
2222 		ret = -EINVAL;  in xhci_evaluate_context_result()
2225 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,  in xhci_evaluate_context_result()
2230 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",  in xhci_evaluate_context_result()
2232 		ret = -EINVAL;  in xhci_evaluate_context_result()
2238 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,  in xhci_count_num_new_endpoints()  argument
2248 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;  in xhci_count_num_new_endpoints()
2249 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;  in xhci_count_num_new_endpoints()
2255 	return hweight32(valid_add_flags) -  in xhci_count_num_new_endpoints()
2259 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,  in xhci_count_num_dropped_endpoints()  argument
2265 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;  in xhci_count_num_dropped_endpoints()
2266 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;  in xhci_count_num_dropped_endpoints()
2268 	return hweight32(valid_drop_flags) -  in xhci_count_num_dropped_endpoints()
2278  *  - the first configure endpoint command drops more endpoints than it adds
2279  *  - a second configure endpoint command that adds more endpoints is queued
2280  *  - the first configure endpoint command fails, so the config is unchanged
2281  *  - the second command may succeed, even though there isn't enough resources
2283  * Must be called with xhci->lock held.
2285 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,  in xhci_reserve_host_resources()  argument
2290 	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);  in xhci_reserve_host_resources()
2291 	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {  in xhci_reserve_host_resources()
2292 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_reserve_host_resources()
2295 				xhci->num_active_eps, added_eps,  in xhci_reserve_host_resources()
2296 				xhci->limit_active_eps);  in xhci_reserve_host_resources()
2297 		return -ENOMEM;  in xhci_reserve_host_resources()
2299 	xhci->num_active_eps += added_eps;  in xhci_reserve_host_resources()
2300 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_reserve_host_resources()
2302 			xhci->num_active_eps);  in xhci_reserve_host_resources()
2310  * Must be called with xhci->lock held.
2312 static void xhci_free_host_resources(struct xhci_hcd *xhci,  in xhci_free_host_resources()  argument
2317 	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);  in xhci_free_host_resources()
2318 	xhci->num_active_eps -= num_failed_eps;  in xhci_free_host_resources()
2319 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_free_host_resources()
2322 			xhci->num_active_eps);  in xhci_free_host_resources()
2329  * Must be called with xhci->lock held.
2331 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,  in xhci_finish_resource_reservation()  argument
2336 	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);  in xhci_finish_resource_reservation()
2337 	xhci->num_active_eps -= num_dropped_eps;  in xhci_finish_resource_reservation()
2339 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_finish_resource_reservation()
2342 				xhci->num_active_eps);  in xhci_finish_resource_reservation()
2347 	switch (udev->speed) {  in xhci_get_block_size()
2367 	if (interval_bw->overhead[LS_OVERHEAD_TYPE])  in xhci_get_largest_overhead()
2369 	if (interval_bw->overhead[FS_OVERHEAD_TYPE])  in xhci_get_largest_overhead()
2378 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,  in xhci_check_tt_bw_table()  argument
2386 	bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;  in xhci_check_tt_bw_table()
2387 	tt_info = virt_dev->tt_info;  in xhci_check_tt_bw_table()
2394 	if (old_active_eps == 0 && tt_info->active_eps != 0) {  in xhci_check_tt_bw_table()
2395 		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)  in xhci_check_tt_bw_table()
2396 			return -ENOMEM;  in xhci_check_tt_bw_table()
2408 static int xhci_check_ss_bw(struct xhci_hcd *xhci,  in xhci_check_ss_bw()  argument
2414 	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))  in xhci_check_ss_bw()
2415 		return -ENOMEM;  in xhci_check_ss_bw()
2418 	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))  in xhci_check_ss_bw()
2419 		return -ENOMEM;  in xhci_check_ss_bw()
2425  * This algorithm is a very conservative estimate of the worst-case scheduling
2433  * over-estimate.
2465 static int xhci_check_bw_table(struct xhci_hcd *xhci,  in xhci_check_bw_table()  argument
2480 	if (virt_dev->udev->speed >= USB_SPEED_SUPER)  in xhci_check_bw_table()
2481 		return xhci_check_ss_bw(xhci, virt_dev);  in xhci_check_bw_table()
2483 	if (virt_dev->udev->speed == USB_SPEED_HIGH) {  in xhci_check_bw_table()
2492 	bw_table = virt_dev->bw_table;  in xhci_check_bw_table()
2496 	block_size = xhci_get_block_size(virt_dev->udev);  in xhci_check_bw_table()
2501 	if (virt_dev->tt_info) {  in xhci_check_bw_table()
2502 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_check_bw_table()
2504 				virt_dev->real_port);  in xhci_check_bw_table()
2505 		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {  in xhci_check_bw_table()
2506 			xhci_warn(xhci, "Not enough bandwidth on HS bus for "  in xhci_check_bw_table()
2508 			return -ENOMEM;  in xhci_check_bw_table()
2510 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_check_bw_table()
2512 				virt_dev->tt_info->slot_id,  in xhci_check_bw_table()
2513 				virt_dev->tt_info->ttport);  in xhci_check_bw_table()
2515 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_check_bw_table()
2517 				virt_dev->real_port);  in xhci_check_bw_table()
2523 	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +  in xhci_check_bw_table()
2524 		bw_table->interval_bw[0].num_packets *  in xhci_check_bw_table()
2525 		xhci_get_largest_overhead(&bw_table->interval_bw[0]);  in xhci_check_bw_table()
2538 			bw_table->interval_bw[i].num_packets;  in xhci_check_bw_table()
2543 		if (list_empty(&bw_table->interval_bw[i].endpoints))  in xhci_check_bw_table()
2549 			ep_entry = bw_table->interval_bw[i].endpoints.next;  in xhci_check_bw_table()
2554 					virt_ep->bw_info.max_packet_size,  in xhci_check_bw_table()
2562 				&bw_table->interval_bw[i]);  in xhci_check_bw_table()
2598 			xhci_warn(xhci, "Not enough bandwidth. "  in xhci_check_bw_table()
2601 			return -ENOMEM;  in xhci_check_bw_table()
2605 	 * Ok, we know we have some packets left over after even-handedly  in xhci_check_bw_table()
2607 	 * fit into, so we over-schedule and say they will be scheduled every  in xhci_check_bw_table()
2613 	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {  in xhci_check_bw_table()
2614 		unsigned int port_index = virt_dev->real_port - 1;  in xhci_check_bw_table()
2621 			xhci->rh_bw[port_index].num_active_tts;  in xhci_check_bw_table()
2624 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_check_bw_table()
2628 		(max_bandwidth - bw_used - bw_reserved) * 100 /  in xhci_check_bw_table()
2633 		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",  in xhci_check_bw_table()
2635 		return -ENOMEM;  in xhci_check_bw_table()
2638 	bw_table->bw_used = bw_used;  in xhci_check_bw_table()
2656 	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);  in xhci_get_ss_bw_consumed()
2658 	if (ep_bw->ep_interval == 0)  in xhci_get_ss_bw_consumed()
2660 			(ep_bw->mult * ep_bw->num_packets *  in xhci_get_ss_bw_consumed()
2662 	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *  in xhci_get_ss_bw_consumed()
2664 				1 << ep_bw->ep_interval);  in xhci_get_ss_bw_consumed()
2668 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,  in xhci_drop_ep_from_interval_table()  argument
2678 	if (xhci_is_async_ep(ep_bw->type))  in xhci_drop_ep_from_interval_table()
2681 	if (udev->speed >= USB_SPEED_SUPER) {  in xhci_drop_ep_from_interval_table()
2682 		if (xhci_is_sync_in_ep(ep_bw->type))  in xhci_drop_ep_from_interval_table()
2683 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=  in xhci_drop_ep_from_interval_table()
2686 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=  in xhci_drop_ep_from_interval_table()
2694 	if (list_empty(&virt_ep->bw_endpoint_list))  in xhci_drop_ep_from_interval_table()
2699 	if (udev->speed == USB_SPEED_HIGH)  in xhci_drop_ep_from_interval_table()
2700 		normalized_interval = ep_bw->ep_interval;  in xhci_drop_ep_from_interval_table()
2702 		normalized_interval = ep_bw->ep_interval - 3;  in xhci_drop_ep_from_interval_table()
2705 		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;  in xhci_drop_ep_from_interval_table()
2706 	interval_bw = &bw_table->interval_bw[normalized_interval];  in xhci_drop_ep_from_interval_table()
2707 	interval_bw->num_packets -= ep_bw->num_packets;  in xhci_drop_ep_from_interval_table()
2708 	switch (udev->speed) {  in xhci_drop_ep_from_interval_table()
2710 		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;  in xhci_drop_ep_from_interval_table()
2713 		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;  in xhci_drop_ep_from_interval_table()
2716 		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;  in xhci_drop_ep_from_interval_table()
2728 		tt_info->active_eps -= 1;  in xhci_drop_ep_from_interval_table()
2729 	list_del_init(&virt_ep->bw_endpoint_list);  in xhci_drop_ep_from_interval_table()
2732 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,  in xhci_add_ep_to_interval_table()  argument
2743 	if (xhci_is_async_ep(ep_bw->type))  in xhci_add_ep_to_interval_table()
2746 	if (udev->speed == USB_SPEED_SUPER) {  in xhci_add_ep_to_interval_table()
2747 		if (xhci_is_sync_in_ep(ep_bw->type))  in xhci_add_ep_to_interval_table()
2748 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=  in xhci_add_ep_to_interval_table()
2751 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=  in xhci_add_ep_to_interval_table()
2759 	if (udev->speed == USB_SPEED_HIGH)  in xhci_add_ep_to_interval_table()
2760 		normalized_interval = ep_bw->ep_interval;  in xhci_add_ep_to_interval_table()
2762 		normalized_interval = ep_bw->ep_interval - 3;  in xhci_add_ep_to_interval_table()
2765 		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;  in xhci_add_ep_to_interval_table()
2766 	interval_bw = &bw_table->interval_bw[normalized_interval];  in xhci_add_ep_to_interval_table()
2767 	interval_bw->num_packets += ep_bw->num_packets;  in xhci_add_ep_to_interval_table()
2768 	switch (udev->speed) {  in xhci_add_ep_to_interval_table()
2770 		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;  in xhci_add_ep_to_interval_table()
2773 		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;  in xhci_add_ep_to_interval_table()
2776 		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;  in xhci_add_ep_to_interval_table()
2789 		tt_info->active_eps += 1;  in xhci_add_ep_to_interval_table()
2791 	list_for_each_entry(smaller_ep, &interval_bw->endpoints,  in xhci_add_ep_to_interval_table()
2793 		if (ep_bw->max_packet_size >=  in xhci_add_ep_to_interval_table()
2794 				smaller_ep->bw_info.max_packet_size) {  in xhci_add_ep_to_interval_table()
2796 			list_add_tail(&virt_ep->bw_endpoint_list,  in xhci_add_ep_to_interval_table()
2797 					&smaller_ep->bw_endpoint_list);  in xhci_add_ep_to_interval_table()
2802 	list_add_tail(&virt_ep->bw_endpoint_list,  in xhci_add_ep_to_interval_table()
2803 			&interval_bw->endpoints);  in xhci_add_ep_to_interval_table()
2806 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,  in xhci_update_tt_active_eps()  argument
2811 	if (!virt_dev->tt_info)  in xhci_update_tt_active_eps()
2814 	rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];  in xhci_update_tt_active_eps()
2816 				virt_dev->tt_info->active_eps != 0) {  in xhci_update_tt_active_eps()
2817 		rh_bw_info->num_active_tts += 1;  in xhci_update_tt_active_eps()
2818 		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;  in xhci_update_tt_active_eps()
2820 				virt_dev->tt_info->active_eps == 0) {  in xhci_update_tt_active_eps()
2821 		rh_bw_info->num_active_tts -= 1;  in xhci_update_tt_active_eps()
2822 		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;  in xhci_update_tt_active_eps()
2826 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,  in xhci_reserve_bandwidth()  argument
2835 	if (virt_dev->tt_info)  in xhci_reserve_bandwidth()
2836 		old_active_eps = virt_dev->tt_info->active_eps;  in xhci_reserve_bandwidth()
2840 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_reserve_bandwidth()
2842 		return -ENOMEM;  in xhci_reserve_bandwidth()
2850 		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,  in xhci_reserve_bandwidth()
2856 			xhci_drop_ep_from_interval_table(xhci,  in xhci_reserve_bandwidth()
2857 					&virt_dev->eps[i].bw_info,  in xhci_reserve_bandwidth()
2858 					virt_dev->bw_table,  in xhci_reserve_bandwidth()
2859 					virt_dev->udev,  in xhci_reserve_bandwidth()
2860 					&virt_dev->eps[i],  in xhci_reserve_bandwidth()
2861 					virt_dev->tt_info);  in xhci_reserve_bandwidth()
2864 	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);  in xhci_reserve_bandwidth()
2868 			xhci_add_ep_to_interval_table(xhci,  in xhci_reserve_bandwidth()
2869 					&virt_dev->eps[i].bw_info,  in xhci_reserve_bandwidth()
2870 					virt_dev->bw_table,  in xhci_reserve_bandwidth()
2871 					virt_dev->udev,  in xhci_reserve_bandwidth()
2872 					&virt_dev->eps[i],  in xhci_reserve_bandwidth()
2873 					virt_dev->tt_info);  in xhci_reserve_bandwidth()
2876 	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {  in xhci_reserve_bandwidth()
2880 		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);  in xhci_reserve_bandwidth()
2893 			xhci_drop_ep_from_interval_table(xhci,  in xhci_reserve_bandwidth()
2894 					&virt_dev->eps[i].bw_info,  in xhci_reserve_bandwidth()
2895 					virt_dev->bw_table,  in xhci_reserve_bandwidth()
2896 					virt_dev->udev,  in xhci_reserve_bandwidth()
2897 					&virt_dev->eps[i],  in xhci_reserve_bandwidth()
2898 					virt_dev->tt_info);  in xhci_reserve_bandwidth()
2901 		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],  in xhci_reserve_bandwidth()
2905 			xhci_add_ep_to_interval_table(xhci,  in xhci_reserve_bandwidth()
2906 					&virt_dev->eps[i].bw_info,  in xhci_reserve_bandwidth()
2907 					virt_dev->bw_table,  in xhci_reserve_bandwidth()
2908 					virt_dev->udev,  in xhci_reserve_bandwidth()
2909 					&virt_dev->eps[i],  in xhci_reserve_bandwidth()
2910 					virt_dev->tt_info);  in xhci_reserve_bandwidth()
2912 	return -ENOMEM;  in xhci_reserve_bandwidth()
2919 static int xhci_configure_endpoint(struct xhci_hcd *xhci,  in xhci_configure_endpoint()  argument
2931 		return -EINVAL;  in xhci_configure_endpoint()
2933 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_configure_endpoint()
2935 	if (xhci->xhc_state & XHCI_STATE_DYING) {  in xhci_configure_endpoint()
2936 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2937 		return -ESHUTDOWN;  in xhci_configure_endpoint()
2940 	virt_dev = xhci->devs[udev->slot_id];  in xhci_configure_endpoint()
2942 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);  in xhci_configure_endpoint()
2944 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2945 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_configure_endpoint()
2947 		return -ENOMEM;  in xhci_configure_endpoint()
2950 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&  in xhci_configure_endpoint()
2951 			xhci_reserve_host_resources(xhci, ctrl_ctx)) {  in xhci_configure_endpoint()
2952 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2953 		xhci_warn(xhci, "Not enough host resources, "  in xhci_configure_endpoint()
2955 				xhci->num_active_eps);  in xhci_configure_endpoint()
2956 		return -ENOMEM;  in xhci_configure_endpoint()
2958 	if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&  in xhci_configure_endpoint()
2959 	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {  in xhci_configure_endpoint()
2960 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))  in xhci_configure_endpoint()
2961 			xhci_free_host_resources(xhci, ctrl_ctx);  in xhci_configure_endpoint()
2962 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2963 		xhci_warn(xhci, "Not enough bandwidth\n");  in xhci_configure_endpoint()
2964 		return -ENOMEM;  in xhci_configure_endpoint()
2967 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);  in xhci_configure_endpoint()
2973 		ret = xhci_queue_configure_endpoint(xhci, command,  in xhci_configure_endpoint()
2974 				command->in_ctx->dma,  in xhci_configure_endpoint()
2975 				udev->slot_id, must_succeed);  in xhci_configure_endpoint()
2977 		ret = xhci_queue_evaluate_context(xhci, command,  in xhci_configure_endpoint()
2978 				command->in_ctx->dma,  in xhci_configure_endpoint()
2979 				udev->slot_id, must_succeed);  in xhci_configure_endpoint()
2981 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))  in xhci_configure_endpoint()
2982 			xhci_free_host_resources(xhci, ctrl_ctx);  in xhci_configure_endpoint()
2983 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2984 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,  in xhci_configure_endpoint()
2986 		return -ENOMEM;  in xhci_configure_endpoint()
2988 	xhci_ring_cmd_db(xhci);  in xhci_configure_endpoint()
2989 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
2992 	wait_for_completion(command->completion);  in xhci_configure_endpoint()
2995 		ret = xhci_configure_endpoint_result(xhci, udev,  in xhci_configure_endpoint()
2996 						     &command->status);  in xhci_configure_endpoint()
2998 		ret = xhci_evaluate_context_result(xhci, udev,  in xhci_configure_endpoint()
2999 						   &command->status);  in xhci_configure_endpoint()
3001 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {  in xhci_configure_endpoint()
3002 		spin_lock_irqsave(&xhci->lock, flags);  in xhci_configure_endpoint()
3007 			xhci_free_host_resources(xhci, ctrl_ctx);  in xhci_configure_endpoint()
3009 			xhci_finish_resource_reservation(xhci, ctrl_ctx);  in xhci_configure_endpoint()
3010 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_configure_endpoint()
3015 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,  in xhci_check_bw_drop_ep_streams()  argument
3018 	struct xhci_virt_ep *ep = &vdev->eps[i];  in xhci_check_bw_drop_ep_streams()
3020 	if (ep->ep_state & EP_HAS_STREAMS) {  in xhci_check_bw_drop_ep_streams()
3021 		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",  in xhci_check_bw_drop_ep_streams()
3023 		xhci_free_stream_info(xhci, ep->stream_info);  in xhci_check_bw_drop_ep_streams()
3024 		ep->stream_info = NULL;  in xhci_check_bw_drop_ep_streams()
3025 		ep->ep_state &= ~EP_HAS_STREAMS;  in xhci_check_bw_drop_ep_streams()
3036  * else should be touching the xhci->devs[slot_id] structure, so we
3037  * don't need to take the xhci->lock for manipulating that.
3043 	struct xhci_hcd *xhci;  in xhci_check_bandwidth()  local
3052 	xhci = hcd_to_xhci(hcd);  in xhci_check_bandwidth()
3053 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||  in xhci_check_bandwidth()
3054 		(xhci->xhc_state & XHCI_STATE_REMOVING))  in xhci_check_bandwidth()
3055 		return -ENODEV;  in xhci_check_bandwidth()
3057 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);  in xhci_check_bandwidth()
3058 	virt_dev = xhci->devs[udev->slot_id];  in xhci_check_bandwidth()
3060 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);  in xhci_check_bandwidth()
3062 		return -ENOMEM;  in xhci_check_bandwidth()
3064 	command->in_ctx = virt_dev->in_ctx;  in xhci_check_bandwidth()
3066 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */  in xhci_check_bandwidth()
3067 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);  in xhci_check_bandwidth()
3069 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_check_bandwidth()
3071 		ret = -ENOMEM;  in xhci_check_bandwidth()
3074 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);  in xhci_check_bandwidth()
3075 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);  in xhci_check_bandwidth()
3076 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));  in xhci_check_bandwidth()
3079 	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&  in xhci_check_bandwidth()
3080 	    ctrl_ctx->drop_flags == 0) {  in xhci_check_bandwidth()
3085 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);  in xhci_check_bandwidth()
3086 	for (i = 31; i >= 1; i--) {  in xhci_check_bandwidth()
3089 		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))  in xhci_check_bandwidth()
3090 		    || (ctrl_ctx->add_flags & le32) || i == 1) {  in xhci_check_bandwidth()
3091 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);  in xhci_check_bandwidth()
3092 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));  in xhci_check_bandwidth()
3097 	ret = xhci_configure_endpoint(xhci, udev, command,  in xhci_check_bandwidth()
3105 		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&  in xhci_check_bandwidth()
3106 		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {  in xhci_check_bandwidth()
3107 			xhci_free_endpoint_ring(xhci, virt_dev, i);  in xhci_check_bandwidth()
3108 			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);  in xhci_check_bandwidth()
3111 	xhci_zero_in_ctx(xhci, virt_dev);  in xhci_check_bandwidth()
3117 		if (!virt_dev->eps[i].new_ring)  in xhci_check_bandwidth()
3122 		if (virt_dev->eps[i].ring) {  in xhci_check_bandwidth()
3123 			xhci_free_endpoint_ring(xhci, virt_dev, i);  in xhci_check_bandwidth()
3125 		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);  in xhci_check_bandwidth()
3126 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;  in xhci_check_bandwidth()
3127 		virt_dev->eps[i].new_ring = NULL;  in xhci_check_bandwidth()
3128 		xhci_debugfs_create_endpoint(xhci, virt_dev, i);  in xhci_check_bandwidth()
3131 	kfree(command->completion);  in xhci_check_bandwidth()
3140 	struct xhci_hcd *xhci;  in xhci_reset_bandwidth()  local
3147 	xhci = hcd_to_xhci(hcd);  in xhci_reset_bandwidth()
3149 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);  in xhci_reset_bandwidth()
3150 	virt_dev = xhci->devs[udev->slot_id];  in xhci_reset_bandwidth()
3153 		if (virt_dev->eps[i].new_ring) {  in xhci_reset_bandwidth()
3154 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);  in xhci_reset_bandwidth()
3155 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);  in xhci_reset_bandwidth()
3156 			virt_dev->eps[i].new_ring = NULL;  in xhci_reset_bandwidth()
3159 	xhci_zero_in_ctx(xhci, virt_dev);  in xhci_reset_bandwidth()
3163 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,  in xhci_setup_input_ctx_for_config_ep()  argument
3169 	ctrl_ctx->add_flags = cpu_to_le32(add_flags);  in xhci_setup_input_ctx_for_config_ep()
3170 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);  in xhci_setup_input_ctx_for_config_ep()
3171 	xhci_slot_copy(xhci, in_ctx, out_ctx);  in xhci_setup_input_ctx_for_config_ep()
3172 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);  in xhci_setup_input_ctx_for_config_ep()
3178 	struct xhci_hcd		*xhci;  in xhci_endpoint_disable()  local
3185 	xhci = hcd_to_xhci(hcd);  in xhci_endpoint_disable()
3187 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_endpoint_disable()
3189 	udev = (struct usb_device *)host_ep->hcpriv;  in xhci_endpoint_disable()
3190 	if (!udev || !udev->slot_id)  in xhci_endpoint_disable()
3193 	vdev = xhci->devs[udev->slot_id];  in xhci_endpoint_disable()
3197 	ep_index = xhci_get_endpoint_index(&host_ep->desc);  in xhci_endpoint_disable()
3198 	ep = &vdev->eps[ep_index];  in xhci_endpoint_disable()
3201 	if (ep->ep_state & EP_CLEARING_TT) {  in xhci_endpoint_disable()
3202 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_disable()
3207 	if (ep->ep_state)  in xhci_endpoint_disable()
3208 		xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",  in xhci_endpoint_disable()
3209 			 ep->ep_state);  in xhci_endpoint_disable()
3211 	host_ep->hcpriv = NULL;  in xhci_endpoint_disable()
3212 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_disable()
3224  * endpoint. Refer to the additional note in xhci spcification section 4.6.8.
3230 	struct xhci_hcd *xhci;  in xhci_endpoint_reset()  local
3241 	xhci = hcd_to_xhci(hcd);  in xhci_endpoint_reset()
3242 	if (!host_ep->hcpriv)  in xhci_endpoint_reset()
3244 	udev = (struct usb_device *) host_ep->hcpriv;  in xhci_endpoint_reset()
3245 	vdev = xhci->devs[udev->slot_id];  in xhci_endpoint_reset()
3248 	 * vdev may be lost due to xHC restore error and re-initialization  in xhci_endpoint_reset()
3252 	if (!udev->slot_id || !vdev)  in xhci_endpoint_reset()
3254 	ep_index = xhci_get_endpoint_index(&host_ep->desc);  in xhci_endpoint_reset()
3255 	ep = &vdev->eps[ep_index];  in xhci_endpoint_reset()
3258 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_endpoint_reset()
3259 	if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {  in xhci_endpoint_reset()
3260 		ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;  in xhci_endpoint_reset()
3261 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3264 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3265 	/* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */  in xhci_endpoint_reset()
3266 	if (usb_endpoint_xfer_control(&host_ep->desc) ||  in xhci_endpoint_reset()
3267 	    usb_endpoint_xfer_isoc(&host_ep->desc))  in xhci_endpoint_reset()
3270 	ep_flag = xhci_get_endpoint_flag(&host_ep->desc);  in xhci_endpoint_reset()
3275 	stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);  in xhci_endpoint_reset()
3279 	cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);  in xhci_endpoint_reset()
3283 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_endpoint_reset()
3286 	ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;  in xhci_endpoint_reset()
3294 	if (!list_empty(&ep->ring->td_list)) {  in xhci_endpoint_reset()
3295 		dev_err(&udev->dev, "EP not empty, refuse reset\n");  in xhci_endpoint_reset()
3296 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3297 		xhci_free_command(xhci, cfg_cmd);  in xhci_endpoint_reset()
3301 	err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,  in xhci_endpoint_reset()
3304 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3305 		xhci_free_command(xhci, cfg_cmd);  in xhci_endpoint_reset()
3306 		xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",  in xhci_endpoint_reset()
3311 	xhci_ring_cmd_db(xhci);  in xhci_endpoint_reset()
3312 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3314 	wait_for_completion(stop_cmd->completion);  in xhci_endpoint_reset()
3316 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_endpoint_reset()
3319 	ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);  in xhci_endpoint_reset()
3321 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3322 		xhci_free_command(xhci, cfg_cmd);  in xhci_endpoint_reset()
3323 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_endpoint_reset()
3328 	xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,  in xhci_endpoint_reset()
3330 	xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);  in xhci_endpoint_reset()
3332 	err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,  in xhci_endpoint_reset()
3333 				      udev->slot_id, false);  in xhci_endpoint_reset()
3335 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3336 		xhci_free_command(xhci, cfg_cmd);  in xhci_endpoint_reset()
3337 		xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",  in xhci_endpoint_reset()
3342 	xhci_ring_cmd_db(xhci);  in xhci_endpoint_reset()
3343 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3345 	wait_for_completion(cfg_cmd->completion);  in xhci_endpoint_reset()
3347 	xhci_free_command(xhci, cfg_cmd);  in xhci_endpoint_reset()
3349 	xhci_free_command(xhci, stop_cmd);  in xhci_endpoint_reset()
3350 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_endpoint_reset()
3351 	if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE)  in xhci_endpoint_reset()
3352 		ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;  in xhci_endpoint_reset()
3353 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_endpoint_reset()
3356 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,  in xhci_check_streams_endpoint()  argument
3365 		return -EINVAL;  in xhci_check_streams_endpoint()
3366 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);  in xhci_check_streams_endpoint()
3368 		return ret ? ret : -EINVAL;  in xhci_check_streams_endpoint()
3369 	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {  in xhci_check_streams_endpoint()
3370 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"  in xhci_check_streams_endpoint()
3372 				ep->desc.bEndpointAddress);  in xhci_check_streams_endpoint()
3373 		return -EINVAL;  in xhci_check_streams_endpoint()
3376 	ep_index = xhci_get_endpoint_index(&ep->desc);  in xhci_check_streams_endpoint()
3377 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;  in xhci_check_streams_endpoint()
3380 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "  in xhci_check_streams_endpoint()
3382 				ep->desc.bEndpointAddress);  in xhci_check_streams_endpoint()
3383 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "  in xhci_check_streams_endpoint()
3385 		return -EINVAL;  in xhci_check_streams_endpoint()
3387 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {  in xhci_check_streams_endpoint()
3388 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "  in xhci_check_streams_endpoint()
3390 				ep->desc.bEndpointAddress);  in xhci_check_streams_endpoint()
3391 		return -EINVAL;  in xhci_check_streams_endpoint()
3396 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,  in xhci_calculate_streams_entries()  argument
3406 	 * level page entries), but that's an optional feature for xHCI host  in xhci_calculate_streams_entries()
3409 	max_streams = HCC_MAX_PSA(xhci->hcc_params);  in xhci_calculate_streams_entries()
3411 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",  in xhci_calculate_streams_entries()
3422 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,  in xhci_calculate_streams_and_bitmask()  argument
3433 		ret = xhci_check_streams_endpoint(xhci, udev,  in xhci_calculate_streams_and_bitmask()
3434 				eps[i], udev->slot_id);  in xhci_calculate_streams_and_bitmask()
3438 		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);  in xhci_calculate_streams_and_bitmask()
3439 		if (max_streams < (*num_streams - 1)) {  in xhci_calculate_streams_and_bitmask()
3440 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",  in xhci_calculate_streams_and_bitmask()
3441 					eps[i]->desc.bEndpointAddress,  in xhci_calculate_streams_and_bitmask()
3446 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);  in xhci_calculate_streams_and_bitmask()
3448 			return -EINVAL;  in xhci_calculate_streams_and_bitmask()
3454 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,  in xhci_calculate_no_streams_bitmask()  argument
3464 	slot_id = udev->slot_id;  in xhci_calculate_no_streams_bitmask()
3465 	if (!xhci->devs[slot_id])  in xhci_calculate_no_streams_bitmask()
3469 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_calculate_no_streams_bitmask()
3470 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;  in xhci_calculate_no_streams_bitmask()
3473 			xhci_warn(xhci, "WARN Can't disable streams for "  in xhci_calculate_no_streams_bitmask()
3476 					eps[i]->desc.bEndpointAddress);  in xhci_calculate_no_streams_bitmask()
3482 			xhci_warn(xhci, "WARN Can't disable streams for "  in xhci_calculate_no_streams_bitmask()
3485 					eps[i]->desc.bEndpointAddress);  in xhci_calculate_no_streams_bitmask()
3486 			xhci_warn(xhci, "WARN xhci_free_streams() called "  in xhci_calculate_no_streams_bitmask()
3487 					"with non-streams endpoint\n");  in xhci_calculate_no_streams_bitmask()
3490 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);  in xhci_calculate_no_streams_bitmask()
3516 	struct xhci_hcd *xhci;  in xhci_alloc_streams()  local
3527 		return -EINVAL;  in xhci_alloc_streams()
3530 	 * stream 0 that is reserved for xHCI usage.  in xhci_alloc_streams()
3533 	xhci = hcd_to_xhci(hcd);  in xhci_alloc_streams()
3534 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",  in xhci_alloc_streams()
3538 	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||  in xhci_alloc_streams()
3539 			HCC_MAX_PSA(xhci->hcc_params) < 4) {  in xhci_alloc_streams()
3540 		xhci_dbg(xhci, "xHCI controller does not support streams.\n");  in xhci_alloc_streams()
3541 		return -ENOSYS;  in xhci_alloc_streams()
3544 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);  in xhci_alloc_streams()
3546 		return -ENOMEM;  in xhci_alloc_streams()
3548 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);  in xhci_alloc_streams()
3550 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_alloc_streams()
3552 		xhci_free_command(xhci, config_cmd);  in xhci_alloc_streams()
3553 		return -ENOMEM;  in xhci_alloc_streams()
3560 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_alloc_streams()
3561 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,  in xhci_alloc_streams()
3564 		xhci_free_command(xhci, config_cmd);  in xhci_alloc_streams()
3565 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_streams()
3569 		xhci_warn(xhci, "WARN: endpoints can't handle "  in xhci_alloc_streams()
3571 		xhci_free_command(xhci, config_cmd);  in xhci_alloc_streams()
3572 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_streams()
3573 		return -EINVAL;  in xhci_alloc_streams()
3575 	vdev = xhci->devs[udev->slot_id];  in xhci_alloc_streams()
3580 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3581 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;  in xhci_alloc_streams()
3583 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_streams()
3589 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);  in xhci_alloc_streams()
3590 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",  in xhci_alloc_streams()
3594 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3595 		max_packet = usb_endpoint_maxp(&eps[i]->desc);  in xhci_alloc_streams()
3596 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,  in xhci_alloc_streams()
3600 		if (!vdev->eps[ep_index].stream_info)  in xhci_alloc_streams()
3611 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3612 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);  in xhci_alloc_streams()
3614 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,  in xhci_alloc_streams()
3615 				vdev->out_ctx, ep_index);  in xhci_alloc_streams()
3616 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,  in xhci_alloc_streams()
3617 				vdev->eps[ep_index].stream_info);  in xhci_alloc_streams()
3622 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,  in xhci_alloc_streams()
3623 			vdev->out_ctx, ctrl_ctx,  in xhci_alloc_streams()
3627 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,  in xhci_alloc_streams()
3637 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_alloc_streams()
3639 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3640 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;  in xhci_alloc_streams()
3641 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",  in xhci_alloc_streams()
3642 			 udev->slot_id, ep_index);  in xhci_alloc_streams()
3643 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;  in xhci_alloc_streams()
3645 	xhci_free_command(xhci, config_cmd);  in xhci_alloc_streams()
3646 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_streams()
3649 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3650 		xhci_debugfs_create_stream_files(xhci, vdev, ep_index);  in xhci_alloc_streams()
3653 	return num_streams - 1;  in xhci_alloc_streams()
3658 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_alloc_streams()
3659 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);  in xhci_alloc_streams()
3660 		vdev->eps[ep_index].stream_info = NULL;  in xhci_alloc_streams()
3664 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;  in xhci_alloc_streams()
3665 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;  in xhci_alloc_streams()
3666 		xhci_endpoint_zero(xhci, vdev, eps[i]);  in xhci_alloc_streams()
3668 	xhci_free_command(xhci, config_cmd);  in xhci_alloc_streams()
3669 	return -ENOMEM;  in xhci_alloc_streams()
3683 	struct xhci_hcd *xhci;  in xhci_free_streams()  local
3691 	xhci = hcd_to_xhci(hcd);  in xhci_free_streams()
3692 	vdev = xhci->devs[udev->slot_id];  in xhci_free_streams()
3695 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_free_streams()
3696 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,  in xhci_free_streams()
3699 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_free_streams()
3700 		return -EINVAL;  in xhci_free_streams()
3707 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);  in xhci_free_streams()
3708 	command = vdev->eps[ep_index].stream_info->free_streams_command;  in xhci_free_streams()
3709 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);  in xhci_free_streams()
3711 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_free_streams()
3712 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_free_streams()
3714 		return -EINVAL;  in xhci_free_streams()
3720 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_free_streams()
3721 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);  in xhci_free_streams()
3722 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=  in xhci_free_streams()
3725 		xhci_endpoint_copy(xhci, command->in_ctx,  in xhci_free_streams()
3726 				vdev->out_ctx, ep_index);  in xhci_free_streams()
3728 				&vdev->eps[ep_index]);  in xhci_free_streams()
3730 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,  in xhci_free_streams()
3731 			vdev->out_ctx, ctrl_ctx,  in xhci_free_streams()
3733 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_free_streams()
3738 	ret = xhci_configure_endpoint(xhci, udev, command,  in xhci_free_streams()
3747 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_free_streams()
3749 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);  in xhci_free_streams()
3750 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);  in xhci_free_streams()
3751 		vdev->eps[ep_index].stream_info = NULL;  in xhci_free_streams()
3755 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;  in xhci_free_streams()
3756 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;  in xhci_free_streams()
3758 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_free_streams()
3768  * Must be called with xhci->lock held.
3770 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,  in xhci_free_device_endpoint_resources()  argument
3778 		if (virt_dev->eps[i].ring) {  in xhci_free_device_endpoint_resources()
3783 	xhci->num_active_eps -= num_dropped_eps;  in xhci_free_device_endpoint_resources()
3785 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_free_device_endpoint_resources()
3789 				xhci->num_active_eps);  in xhci_free_device_endpoint_resources()
3796  * xhci_address_device(), and then re-set up the configuration.  If this is
3798  * settings will be re-installed through the normal bandwidth allocation
3807  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3808  * re-allocate the device.
3815 	struct xhci_hcd *xhci;  in xhci_discover_or_reset_device()  local
3825 	xhci = hcd_to_xhci(hcd);  in xhci_discover_or_reset_device()
3826 	slot_id = udev->slot_id;  in xhci_discover_or_reset_device()
3827 	virt_dev = xhci->devs[slot_id];  in xhci_discover_or_reset_device()
3829 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "  in xhci_discover_or_reset_device()
3830 				"not exist. Re-allocate the device\n", slot_id);  in xhci_discover_or_reset_device()
3835 			return -EINVAL;  in xhci_discover_or_reset_device()
3838 	if (virt_dev->tt_info)  in xhci_discover_or_reset_device()
3839 		old_active_eps = virt_dev->tt_info->active_eps;  in xhci_discover_or_reset_device()
3841 	if (virt_dev->udev != udev) {  in xhci_discover_or_reset_device()
3844 		 * Re-allocate the device.  in xhci_discover_or_reset_device()
3846 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "  in xhci_discover_or_reset_device()
3847 				"not match the udev. Re-allocate the device\n",  in xhci_discover_or_reset_device()
3853 			return -EINVAL;  in xhci_discover_or_reset_device()
3857 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);  in xhci_discover_or_reset_device()
3858 	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==  in xhci_discover_or_reset_device()
3864 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);  in xhci_discover_or_reset_device()
3871 	reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);  in xhci_discover_or_reset_device()
3873 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");  in xhci_discover_or_reset_device()
3874 		return -ENOMEM;  in xhci_discover_or_reset_device()
3878 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_discover_or_reset_device()
3880 	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);  in xhci_discover_or_reset_device()
3882 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");  in xhci_discover_or_reset_device()
3883 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_discover_or_reset_device()
3886 	xhci_ring_cmd_db(xhci);  in xhci_discover_or_reset_device()
3887 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_discover_or_reset_device()
3890 	wait_for_completion(reset_device_cmd->completion);  in xhci_discover_or_reset_device()
3896 	ret = reset_device_cmd->status;  in xhci_discover_or_reset_device()
3900 		xhci_warn(xhci, "Timeout waiting for reset device command\n");  in xhci_discover_or_reset_device()
3901 		ret = -ETIME;  in xhci_discover_or_reset_device()
3905 		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",  in xhci_discover_or_reset_device()
3907 				xhci_get_slot_state(xhci, virt_dev->out_ctx));  in xhci_discover_or_reset_device()
3908 		xhci_dbg(xhci, "Not freeing device rings.\n");  in xhci_discover_or_reset_device()
3913 		xhci_dbg(xhci, "Successful reset device command.\n");  in xhci_discover_or_reset_device()
3916 		if (xhci_is_vendor_info_code(xhci, ret))  in xhci_discover_or_reset_device()
3918 		xhci_warn(xhci, "Unknown completion code %u for "  in xhci_discover_or_reset_device()
3920 		ret = -EINVAL;  in xhci_discover_or_reset_device()
3925 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {  in xhci_discover_or_reset_device()
3926 		spin_lock_irqsave(&xhci->lock, flags);  in xhci_discover_or_reset_device()
3928 		xhci_free_device_endpoint_resources(xhci, virt_dev, false);  in xhci_discover_or_reset_device()
3929 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_discover_or_reset_device()
3934 		struct xhci_virt_ep *ep = &virt_dev->eps[i];  in xhci_discover_or_reset_device()
3936 		if (ep->ep_state & EP_HAS_STREAMS) {  in xhci_discover_or_reset_device()
3937 			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",  in xhci_discover_or_reset_device()
3939 			xhci_free_stream_info(xhci, ep->stream_info);  in xhci_discover_or_reset_device()
3940 			ep->stream_info = NULL;  in xhci_discover_or_reset_device()
3941 			ep->ep_state &= ~EP_HAS_STREAMS;  in xhci_discover_or_reset_device()
3944 		if (ep->ring) {  in xhci_discover_or_reset_device()
3945 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);  in xhci_discover_or_reset_device()
3946 			xhci_free_endpoint_ring(xhci, virt_dev, i);  in xhci_discover_or_reset_device()
3948 		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))  in xhci_discover_or_reset_device()
3949 			xhci_drop_ep_from_interval_table(xhci,  in xhci_discover_or_reset_device()
3950 					&virt_dev->eps[i].bw_info,  in xhci_discover_or_reset_device()
3951 					virt_dev->bw_table,  in xhci_discover_or_reset_device()
3953 					&virt_dev->eps[i],  in xhci_discover_or_reset_device()
3954 					virt_dev->tt_info);  in xhci_discover_or_reset_device()
3955 		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);  in xhci_discover_or_reset_device()
3958 	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);  in xhci_discover_or_reset_device()
3959 	virt_dev->flags = 0;  in xhci_discover_or_reset_device()
3963 	xhci_free_command(xhci, reset_device_cmd);  in xhci_discover_or_reset_device()
3974 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_free_dev()  local
3984 	if (xhci->quirks & XHCI_RESET_ON_RESUME)  in xhci_free_dev()
3985 		pm_runtime_put_noidle(hcd->self.controller);  in xhci_free_dev()
3991 	if (ret <= 0 && ret != -ENODEV)  in xhci_free_dev()
3994 	virt_dev = xhci->devs[udev->slot_id];  in xhci_free_dev()
3995 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);  in xhci_free_dev()
4000 		virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;  in xhci_free_dev()
4001 	virt_dev->udev = NULL;  in xhci_free_dev()
4002 	xhci_disable_slot(xhci, udev->slot_id);  in xhci_free_dev()
4003 	xhci_free_virt_device(xhci, udev->slot_id);  in xhci_free_dev()
4006 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)  in xhci_disable_slot()  argument
4013 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);  in xhci_disable_slot()
4015 		return -ENOMEM;  in xhci_disable_slot()
4017 	xhci_debugfs_remove_slot(xhci, slot_id);  in xhci_disable_slot()
4019 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_disable_slot()
4021 	state = readl(&xhci->op_regs->status);  in xhci_disable_slot()
4022 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||  in xhci_disable_slot()
4023 			(xhci->xhc_state & XHCI_STATE_HALTED)) {  in xhci_disable_slot()
4024 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_disable_slot()
4026 		return -ENODEV;  in xhci_disable_slot()
4029 	ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,  in xhci_disable_slot()
4032 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_disable_slot()
4036 	xhci_ring_cmd_db(xhci);  in xhci_disable_slot()
4037 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_disable_slot()
4039 	wait_for_completion(command->completion);  in xhci_disable_slot()
4041 	if (command->status != COMP_SUCCESS)  in xhci_disable_slot()
4042 		xhci_warn(xhci, "Unsuccessful disable slot %u command, status %d\n",  in xhci_disable_slot()
4043 			  slot_id, command->status);  in xhci_disable_slot()
4045 	xhci_free_command(xhci, command);  in xhci_disable_slot()
4054  * Must be called with xhci->lock held.
4056 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)  in xhci_reserve_host_control_ep_resources()  argument
4058 	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {  in xhci_reserve_host_control_ep_resources()
4059 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_reserve_host_control_ep_resources()
4062 				xhci->num_active_eps, xhci->limit_active_eps);  in xhci_reserve_host_control_ep_resources()
4063 		return -ENOMEM;  in xhci_reserve_host_control_ep_resources()
4065 	xhci->num_active_eps += 1;  in xhci_reserve_host_control_ep_resources()
4066 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,  in xhci_reserve_host_control_ep_resources()
4068 			xhci->num_active_eps);  in xhci_reserve_host_control_ep_resources()
4079 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_alloc_dev()  local
4086 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);  in xhci_alloc_dev()
4090 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_alloc_dev()
4091 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);  in xhci_alloc_dev()
4093 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_dev()
4094 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");  in xhci_alloc_dev()
4095 		xhci_free_command(xhci, command);  in xhci_alloc_dev()
4098 	xhci_ring_cmd_db(xhci);  in xhci_alloc_dev()
4099 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_dev()
4101 	wait_for_completion(command->completion);  in xhci_alloc_dev()
4102 	slot_id = command->slot_id;  in xhci_alloc_dev()
4104 	if (!slot_id || command->status != COMP_SUCCESS) {  in xhci_alloc_dev()
4105 		xhci_err(xhci, "Error while assigning device slot ID: %s\n",  in xhci_alloc_dev()
4106 			 xhci_trb_comp_code_string(command->status));  in xhci_alloc_dev()
4107 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",  in xhci_alloc_dev()
4109 					readl(&xhci->cap_regs->hcs_params1)));  in xhci_alloc_dev()
4110 		xhci_free_command(xhci, command);  in xhci_alloc_dev()
4114 	xhci_free_command(xhci, command);  in xhci_alloc_dev()
4116 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {  in xhci_alloc_dev()
4117 		spin_lock_irqsave(&xhci->lock, flags);  in xhci_alloc_dev()
4118 		ret = xhci_reserve_host_control_ep_resources(xhci);  in xhci_alloc_dev()
4120 			spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_dev()
4121 			xhci_warn(xhci, "Not enough host resources, "  in xhci_alloc_dev()
4123 					xhci->num_active_eps);  in xhci_alloc_dev()
4126 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_alloc_dev()
4132 	if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {  in xhci_alloc_dev()
4133 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");  in xhci_alloc_dev()
4136 	vdev = xhci->devs[slot_id];  in xhci_alloc_dev()
4137 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);  in xhci_alloc_dev()
4140 	udev->slot_id = slot_id;  in xhci_alloc_dev()
4142 	xhci_debugfs_create_slot(xhci, slot_id);  in xhci_alloc_dev()
4148 	if (xhci->quirks & XHCI_RESET_ON_RESUME)  in xhci_alloc_dev()
4149 		pm_runtime_get_noresume(hcd->self.controller);  in xhci_alloc_dev()
4156 	xhci_disable_slot(xhci, udev->slot_id);  in xhci_alloc_dev()
4157 	xhci_free_virt_device(xhci, udev->slot_id);  in xhci_alloc_dev()
4173 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_setup_device()  local
4179 	mutex_lock(&xhci->mutex);  in xhci_setup_device()
4181 	if (xhci->xhc_state) {	/* dying, removing or halted */  in xhci_setup_device()
4182 		ret = -ESHUTDOWN;  in xhci_setup_device()
4186 	if (!udev->slot_id) {  in xhci_setup_device()
4187 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4188 				"Bad Slot ID %d", udev->slot_id);  in xhci_setup_device()
4189 		ret = -EINVAL;  in xhci_setup_device()
4193 	virt_dev = xhci->devs[udev->slot_id];  in xhci_setup_device()
4198 		 * a zero-dereference was observed once due to virt_dev = 0.  in xhci_setup_device()
4201 		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",  in xhci_setup_device()
4202 			udev->slot_id);  in xhci_setup_device()
4203 		ret = -EINVAL;  in xhci_setup_device()
4206 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);  in xhci_setup_device()
4210 		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==  in xhci_setup_device()
4212 			xhci_dbg(xhci, "Slot already in default state\n");  in xhci_setup_device()
4217 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);  in xhci_setup_device()
4219 		ret = -ENOMEM;  in xhci_setup_device()
4223 	command->in_ctx = virt_dev->in_ctx;  in xhci_setup_device()
4225 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);  in xhci_setup_device()
4226 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);  in xhci_setup_device()
4228 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_setup_device()
4230 		ret = -EINVAL;  in xhci_setup_device()
4234 	 * If this is the first Set Address since device plug-in or  in xhci_setup_device()
4235 	 * virt_device realloaction after a resume with an xHCI power loss,  in xhci_setup_device()
4238 	if (!slot_ctx->dev_info)  in xhci_setup_device()
4239 		xhci_setup_addressable_virt_dev(xhci, udev);  in xhci_setup_device()
4242 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);  in xhci_setup_device()
4243 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);  in xhci_setup_device()
4244 	ctrl_ctx->drop_flags = 0;  in xhci_setup_device()
4246 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,  in xhci_setup_device()
4247 				le32_to_cpu(slot_ctx->dev_info) >> 27);  in xhci_setup_device()
4250 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_setup_device()
4252 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,  in xhci_setup_device()
4253 					udev->slot_id, setup);  in xhci_setup_device()
4255 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_setup_device()
4256 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4260 	xhci_ring_cmd_db(xhci);  in xhci_setup_device()
4261 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_setup_device()
4264 	wait_for_completion(command->completion);  in xhci_setup_device()
4270 	switch (command->status) {  in xhci_setup_device()
4273 		xhci_warn(xhci, "Timeout while waiting for setup device command\n");  in xhci_setup_device()
4274 		ret = -ETIME;  in xhci_setup_device()
4278 		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",  in xhci_setup_device()
4279 			 act, udev->slot_id);  in xhci_setup_device()
4280 		ret = -EINVAL;  in xhci_setup_device()
4283 		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);  in xhci_setup_device()
4285 		mutex_unlock(&xhci->mutex);  in xhci_setup_device()
4286 		ret = xhci_disable_slot(xhci, udev->slot_id);  in xhci_setup_device()
4287 		xhci_free_virt_device(xhci, udev->slot_id);  in xhci_setup_device()
4290 		kfree(command->completion);  in xhci_setup_device()
4292 		return -EPROTO;  in xhci_setup_device()
4294 		dev_warn(&udev->dev,  in xhci_setup_device()
4296 		ret = -ENODEV;  in xhci_setup_device()
4299 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4303 		xhci_err(xhci,  in xhci_setup_device()
4305 			 act, command->status);  in xhci_setup_device()
4306 		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);  in xhci_setup_device()
4307 		ret = -EINVAL;  in xhci_setup_device()
4312 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);  in xhci_setup_device()
4313 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4315 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4317 		udev->slot_id,  in xhci_setup_device()
4318 		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],  in xhci_setup_device()
4320 		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));  in xhci_setup_device()
4321 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4323 			(unsigned long long)virt_dev->out_ctx->dma);  in xhci_setup_device()
4324 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,  in xhci_setup_device()
4325 				le32_to_cpu(slot_ctx->dev_info) >> 27);  in xhci_setup_device()
4330 	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,  in xhci_setup_device()
4331 				le32_to_cpu(slot_ctx->dev_info) >> 27);  in xhci_setup_device()
4333 	ctrl_ctx->add_flags = 0;  in xhci_setup_device()
4334 	ctrl_ctx->drop_flags = 0;  in xhci_setup_device()
4335 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);  in xhci_setup_device()
4336 	udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);  in xhci_setup_device()
4338 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,  in xhci_setup_device()
4340 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);  in xhci_setup_device()
4342 	mutex_unlock(&xhci->mutex);  in xhci_setup_device()
4344 		kfree(command->completion);  in xhci_setup_device()
4371 	return rhub->ports[port1 - 1]->hw_portnum + 1;  in xhci_find_raw_port_number()
4378 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,  in xhci_change_max_exit_latency()  argument
4388 	command = xhci_alloc_command_with_ctx(xhci, true, GFP_KERNEL);  in xhci_change_max_exit_latency()
4390 		return -ENOMEM;  in xhci_change_max_exit_latency()
4392 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4394 	virt_dev = xhci->devs[udev->slot_id];  in xhci_change_max_exit_latency()
4398 	 * xHC was re-initialized. Exit latency will be set later after  in xhci_change_max_exit_latency()
4399 	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated  in xhci_change_max_exit_latency()
4402 	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {  in xhci_change_max_exit_latency()
4403 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4408 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);  in xhci_change_max_exit_latency()
4410 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4411 		xhci_free_command(xhci, command);  in xhci_change_max_exit_latency()
4412 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_change_max_exit_latency()
4414 		return -ENOMEM;  in xhci_change_max_exit_latency()
4417 	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);  in xhci_change_max_exit_latency()
4418 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4420 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);  in xhci_change_max_exit_latency()
4421 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);  in xhci_change_max_exit_latency()
4422 	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));  in xhci_change_max_exit_latency()
4423 	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);  in xhci_change_max_exit_latency()
4424 	slot_ctx->dev_state = 0;  in xhci_change_max_exit_latency()
4426 	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,  in xhci_change_max_exit_latency()
4430 	ret = xhci_configure_endpoint(xhci, udev, command,  in xhci_change_max_exit_latency()
4434 		spin_lock_irqsave(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4435 		virt_dev->current_mel = max_exit_latency;  in xhci_change_max_exit_latency()
4436 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_change_max_exit_latency()
4439 	xhci_free_command(xhci, command);  in xhci_change_max_exit_latency()
4451 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,  in xhci_calculate_hird_besl()  argument
4458 	u2del = HCS_U2_LATENCY(xhci->hcs_params3);  in xhci_calculate_hird_besl()
4459 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);  in xhci_calculate_hird_besl()
4475 			besl_host = (u2del - 51) / 75 + 1;  in xhci_calculate_hird_besl()
4493 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);  in xhci_calculate_usb2_hw_lpm_params()
4495 	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */  in xhci_calculate_usb2_hw_lpm_params()
4496 	l1 = udev->l1_params.timeout / 256;  in xhci_calculate_usb2_hw_lpm_params()
4510 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);  in xhci_set_usb2_hardware_lpm()  local
4519 	if (xhci->quirks & XHCI_HW_LPM_DISABLE)  in xhci_set_usb2_hardware_lpm()
4520 		return -EPERM;  in xhci_set_usb2_hardware_lpm()
4522 	if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||  in xhci_set_usb2_hardware_lpm()
4523 			!udev->lpm_capable)  in xhci_set_usb2_hardware_lpm()
4524 		return -EPERM;  in xhci_set_usb2_hardware_lpm()
4526 	if (!udev->parent || udev->parent->parent ||  in xhci_set_usb2_hardware_lpm()
4527 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)  in xhci_set_usb2_hardware_lpm()
4528 		return -EPERM;  in xhci_set_usb2_hardware_lpm()
4530 	if (udev->usb2_hw_lpm_capable != 1)  in xhci_set_usb2_hardware_lpm()
4531 		return -EPERM;  in xhci_set_usb2_hardware_lpm()
4533 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_set_usb2_hardware_lpm()
4535 	ports = xhci->usb2_rhub.ports;  in xhci_set_usb2_hardware_lpm()
4536 	port_num = udev->portnum - 1;  in xhci_set_usb2_hardware_lpm()
4537 	pm_addr = ports[port_num]->addr + PORTPMSC;  in xhci_set_usb2_hardware_lpm()
4539 	hlpm_addr = ports[port_num]->addr + PORTHLPMC;  in xhci_set_usb2_hardware_lpm()
4541 	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",  in xhci_set_usb2_hardware_lpm()
4546 		if (udev->usb2_hw_lpm_besl_capable) {  in xhci_set_usb2_hardware_lpm()
4549 			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h  in xhci_set_usb2_hardware_lpm()
4551 			field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);  in xhci_set_usb2_hardware_lpm()
4556 				hird = udev->l1_params.besl;  in xhci_set_usb2_hardware_lpm()
4559 			spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_set_usb2_hardware_lpm()
4561 			ret = xhci_change_max_exit_latency(xhci, udev,  in xhci_set_usb2_hardware_lpm()
4565 			spin_lock_irqsave(&xhci->lock, flags);  in xhci_set_usb2_hardware_lpm()
4572 			hird = xhci_calculate_hird_besl(xhci, udev);  in xhci_set_usb2_hardware_lpm()
4576 		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);  in xhci_set_usb2_hardware_lpm()
4588 		if (udev->usb2_hw_lpm_besl_capable) {  in xhci_set_usb2_hardware_lpm()
4589 			spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_set_usb2_hardware_lpm()
4590 			xhci_change_max_exit_latency(xhci, udev, 0);  in xhci_set_usb2_hardware_lpm()
4591 			readl_poll_timeout(ports[port_num]->addr, pm_val,  in xhci_set_usb2_hardware_lpm()
4598 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_set_usb2_hardware_lpm()
4606 static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,  in xhci_check_usb2_port_capability()  argument
4612 	for (i = 0; i < xhci->num_ext_caps; i++) {  in xhci_check_usb2_port_capability()
4613 		if (xhci->ext_caps[i] & capability) {  in xhci_check_usb2_port_capability()
4615 			port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;  in xhci_check_usb2_port_capability()
4616 			port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);  in xhci_check_usb2_port_capability()
4627 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);  in xhci_update_device()  local
4628 	int		portnum = udev->portnum - 1;  in xhci_update_device()
4630 	if (hcd->speed >= HCD_USB3 || !udev->lpm_capable)  in xhci_update_device()
4633 	/* we only support lpm for non-hub device connected to root hub yet */  in xhci_update_device()
4634 	if (!udev->parent || udev->parent->parent ||  in xhci_update_device()
4635 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)  in xhci_update_device()
4638 	if (xhci->hw_lpm_support == 1 &&  in xhci_update_device()
4640 				xhci, portnum, XHCI_HLC)) {  in xhci_update_device()
4641 		udev->usb2_hw_lpm_capable = 1;  in xhci_update_device()
4642 		udev->l1_params.timeout = XHCI_L1_TIMEOUT;  in xhci_update_device()
4643 		udev->l1_params.besl = XHCI_DEFAULT_BESL;  in xhci_update_device()
4644 		if (xhci_check_usb2_port_capability(xhci, portnum,  in xhci_update_device()
4646 			udev->usb2_hw_lpm_besl_capable = 1;  in xhci_update_device()
4652 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4654 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4658 	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;  in xhci_service_interval_to_ns()
4672 		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);  in xhci_get_timeout_no_hub_lpm()
4673 		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);  in xhci_get_timeout_no_hub_lpm()
4678 		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);  in xhci_get_timeout_no_hub_lpm()
4679 		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);  in xhci_get_timeout_no_hub_lpm()
4684 		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",  in xhci_get_timeout_no_hub_lpm()
4693 		dev_dbg(&udev->dev, "Device-initiated %s disabled "  in xhci_get_timeout_no_hub_lpm()
4697 		dev_dbg(&udev->dev, "Device-initiated %s disabled "  in xhci_get_timeout_no_hub_lpm()
4704  *  - For control endpoints, U1 system exit latency (SEL) * 3
4705  *  - For bulk endpoints, U1 SEL * 5
4706  *  - For interrupt endpoints:
4707  *    - Notification EPs, U1 SEL * 3
4708  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4709  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4722 		timeout_ns = udev->u1_params.sel * 3;  in xhci_calculate_intel_u1_timeout()
4725 		timeout_ns = udev->u1_params.sel * 5;  in xhci_calculate_intel_u1_timeout()
4730 			timeout_ns = udev->u1_params.sel * 3;  in xhci_calculate_intel_u1_timeout()
4738 		if (timeout_ns < udev->u1_params.sel * 2)  in xhci_calculate_intel_u1_timeout()
4739 			timeout_ns = udev->u1_params.sel * 2;  in xhci_calculate_intel_u1_timeout()
4748 /* Returns the hub-encoded U1 timeout value. */
4749 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,  in xhci_calculate_u1_timeout()  argument
4757 		if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {  in xhci_calculate_u1_timeout()
4758 			dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");  in xhci_calculate_u1_timeout()
4763 	if (xhci->quirks & XHCI_INTEL_HOST)  in xhci_calculate_u1_timeout()
4766 		timeout_ns = udev->u1_params.sel;  in xhci_calculate_u1_timeout()
4777 	 * USB 3.0 hub, we have to disable hub-initiated U1.  in xhci_calculate_u1_timeout()
4781 	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "  in xhci_calculate_u1_timeout()
4787  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4788  *  - largest bInterval of any active periodic endpoint (to avoid going
4790  *  - the U2 Exit Latency of the device
4805 	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;  in xhci_calculate_intel_u2_timeout()
4812 /* Returns the hub-encoded U2 timeout value. */
4813 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,  in xhci_calculate_u2_timeout()  argument
4821 		if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {  in xhci_calculate_u2_timeout()
4822 			dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");  in xhci_calculate_u2_timeout()
4827 	if (xhci->quirks & XHCI_INTEL_HOST)  in xhci_calculate_u2_timeout()
4830 		timeout_ns = udev->u2_params.sel;  in xhci_calculate_u2_timeout()
4835 	 * USB 3.0 hub, we have to disable hub-initiated U2.  in xhci_calculate_u2_timeout()
4839 	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "  in xhci_calculate_u2_timeout()
4844 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,  in xhci_call_host_update_timeout_for_endpoint()  argument
4851 		return xhci_calculate_u1_timeout(xhci, udev, desc);  in xhci_call_host_update_timeout_for_endpoint()
4853 		return xhci_calculate_u2_timeout(xhci, udev, desc);  in xhci_call_host_update_timeout_for_endpoint()
4858 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,  in xhci_update_timeout_for_endpoint()  argument
4866 	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,  in xhci_update_timeout_for_endpoint()
4869 	/* If we found we can't enable hub-initiated LPM, and  in xhci_update_timeout_for_endpoint()
4871 	 * device-initiated LPM as well, then we will disable LPM  in xhci_update_timeout_for_endpoint()
4876 		return -E2BIG;  in xhci_update_timeout_for_endpoint()
4883 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,  in xhci_update_timeout_for_interface()  argument
4891 	for (j = 0; j < alt->desc.bNumEndpoints; j++) {  in xhci_update_timeout_for_interface()
4892 		if (xhci_update_timeout_for_endpoint(xhci, udev,  in xhci_update_timeout_for_interface()
4893 					&alt->endpoint[j].desc, state, timeout))  in xhci_update_timeout_for_interface()
4894 			return -E2BIG;  in xhci_update_timeout_for_interface()
4906 	for (parent = udev->parent, num_hubs = 0; parent->parent;  in xhci_check_intel_tier_policy()
4907 			parent = parent->parent)  in xhci_check_intel_tier_policy()
4913 	dev_dbg(&udev->dev, "Disabling U1/U2 link state for device"  in xhci_check_intel_tier_policy()
4914 			" below second-tier hub.\n");  in xhci_check_intel_tier_policy()
4915 	dev_dbg(&udev->dev, "Plug device into first-tier hub "  in xhci_check_intel_tier_policy()
4917 	return -E2BIG;  in xhci_check_intel_tier_policy()
4920 static int xhci_check_tier_policy(struct xhci_hcd *xhci,  in xhci_check_tier_policy()  argument
4924 	if (xhci->quirks & XHCI_INTEL_HOST)  in xhci_check_tier_policy()
4931  * If the tier check or timeout setting functions return with a non-zero exit
4938 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_calculate_lpm_timeout()  local
4949 		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",  in xhci_calculate_lpm_timeout()
4957 	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,  in xhci_calculate_lpm_timeout()
4961 	config = udev->actconfig;  in xhci_calculate_lpm_timeout()
4965 	for (i = 0; i < config->desc.bNumInterfaces; i++) {  in xhci_calculate_lpm_timeout()
4967 		struct usb_interface *intf = config->interface[i];  in xhci_calculate_lpm_timeout()
4972 		/* Check if any currently bound drivers want hub-initiated LPM  in xhci_calculate_lpm_timeout()
4975 		if (intf->dev.driver) {  in xhci_calculate_lpm_timeout()
4976 			driver = to_usb_driver(intf->dev.driver);  in xhci_calculate_lpm_timeout()
4977 			if (driver && driver->disable_hub_initiated_lpm) {  in xhci_calculate_lpm_timeout()
4978 				dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",  in xhci_calculate_lpm_timeout()
4979 					state_name, driver->name);  in xhci_calculate_lpm_timeout()
4988 		if (!intf->cur_altsetting)  in xhci_calculate_lpm_timeout()
4991 		if (xhci_update_timeout_for_interface(xhci, udev,  in xhci_calculate_lpm_timeout()
4992 					intf->cur_altsetting,  in xhci_calculate_lpm_timeout()
5024 	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||  in calculate_max_exit_latency()
5026 		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);  in calculate_max_exit_latency()
5027 	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||  in calculate_max_exit_latency()
5029 		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);  in calculate_max_exit_latency()
5033 	/* xHCI host controller max exit latency field is only 16 bits wide. */  in calculate_max_exit_latency()
5035 		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "  in calculate_max_exit_latency()
5037 		return -E2BIG;  in calculate_max_exit_latency()
5042 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
5046 	struct xhci_hcd	*xhci;  in xhci_enable_usb3_lpm_timeout()  local
5051 	xhci = hcd_to_xhci(hcd);  in xhci_enable_usb3_lpm_timeout()
5052 	/* The LPM timeout values are pretty host-controller specific, so don't  in xhci_enable_usb3_lpm_timeout()
5053 	 * enable hub-initiated timeouts unless the vendor has provided  in xhci_enable_usb3_lpm_timeout()
5056 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||  in xhci_enable_usb3_lpm_timeout()
5057 			!xhci->devs[udev->slot_id])  in xhci_enable_usb3_lpm_timeout()
5060 	if (xhci_check_tier_policy(xhci, udev, state) < 0)  in xhci_enable_usb3_lpm_timeout()
5071 	ret = xhci_change_max_exit_latency(xhci, udev, mel);  in xhci_enable_usb3_lpm_timeout()
5080 	struct xhci_hcd	*xhci;  in xhci_disable_usb3_lpm_timeout()  local
5083 	xhci = hcd_to_xhci(hcd);  in xhci_disable_usb3_lpm_timeout()
5084 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||  in xhci_disable_usb3_lpm_timeout()
5085 			!xhci->devs[udev->slot_id])  in xhci_disable_usb3_lpm_timeout()
5089 	return xhci_change_max_exit_latency(xhci, udev, mel);  in xhci_disable_usb3_lpm_timeout()
5117 /*-------------------------------------------------------------------------*/
5125 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_update_hub_device()  local
5135 	if (!hdev->parent)  in xhci_update_hub_device()
5138 	vdev = xhci->devs[hdev->slot_id];  in xhci_update_hub_device()
5140 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");  in xhci_update_hub_device()
5141 		return -EINVAL;  in xhci_update_hub_device()
5144 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);  in xhci_update_hub_device()
5146 		return -ENOMEM;  in xhci_update_hub_device()
5148 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);  in xhci_update_hub_device()
5150 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",  in xhci_update_hub_device()
5152 		xhci_free_command(xhci, config_cmd);  in xhci_update_hub_device()
5153 		return -ENOMEM;  in xhci_update_hub_device()
5156 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_update_hub_device()
5157 	if (hdev->speed == USB_SPEED_HIGH &&  in xhci_update_hub_device()
5158 			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {  in xhci_update_hub_device()
5159 		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");  in xhci_update_hub_device()
5160 		xhci_free_command(xhci, config_cmd);  in xhci_update_hub_device()
5161 		spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_update_hub_device()
5162 		return -ENOMEM;  in xhci_update_hub_device()
5165 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);  in xhci_update_hub_device()
5166 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);  in xhci_update_hub_device()
5167 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);  in xhci_update_hub_device()
5168 	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);  in xhci_update_hub_device()
5171 	 * but it may be already set to 1 when setup an xHCI virtual  in xhci_update_hub_device()
5174 	if (tt->multi)  in xhci_update_hub_device()
5175 		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);  in xhci_update_hub_device()
5176 	else if (hdev->speed == USB_SPEED_FULL)  in xhci_update_hub_device()
5177 		slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);  in xhci_update_hub_device()
5179 	if (xhci->hci_version > 0x95) {  in xhci_update_hub_device()
5180 		xhci_dbg(xhci, "xHCI version %x needs hub "  in xhci_update_hub_device()
5182 				(unsigned int) xhci->hci_version);  in xhci_update_hub_device()
5183 		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));  in xhci_update_hub_device()
5184 		/* Set TT think time - convert from ns to FS bit times.  in xhci_update_hub_device()
5188 		 * xHCI 1.0: this field shall be 0 if the device is not a  in xhci_update_hub_device()
5189 		 * High-spped hub.  in xhci_update_hub_device()
5191 		think_time = tt->think_time;  in xhci_update_hub_device()
5193 			think_time = (think_time / 666) - 1;  in xhci_update_hub_device()
5194 		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)  in xhci_update_hub_device()
5195 			slot_ctx->tt_info |=  in xhci_update_hub_device()
5198 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "  in xhci_update_hub_device()
5200 				(unsigned int) xhci->hci_version);  in xhci_update_hub_device()
5202 	slot_ctx->dev_state = 0;  in xhci_update_hub_device()
5203 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_update_hub_device()
5205 	xhci_dbg(xhci, "Set up %s for hub device.\n",  in xhci_update_hub_device()
5206 			(xhci->hci_version > 0x95) ?  in xhci_update_hub_device()
5212 	if (xhci->hci_version > 0x95)  in xhci_update_hub_device()
5213 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,  in xhci_update_hub_device()
5216 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,  in xhci_update_hub_device()
5219 	xhci_free_command(xhci, config_cmd);  in xhci_update_hub_device()
5225 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);  in xhci_get_frame()  local
5227 	return readl(&xhci->run_regs->microframe_index) >> 3;  in xhci_get_frame()
5230 static void xhci_hcd_init_usb2_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)  in xhci_hcd_init_usb2_data()  argument
5232 	xhci->usb2_rhub.hcd = hcd;  in xhci_hcd_init_usb2_data()
5233 	hcd->speed = HCD_USB2;  in xhci_hcd_init_usb2_data()
5234 	hcd->self.root_hub->speed = USB_SPEED_HIGH;  in xhci_hcd_init_usb2_data()
5236 	 * USB 2.0 roothub under xHCI has an integrated TT,  in xhci_hcd_init_usb2_data()
5240 	hcd->has_tt = 1;  in xhci_hcd_init_usb2_data()
5243 static void xhci_hcd_init_usb3_data(struct xhci_hcd *xhci, struct usb_hcd *hcd)  in xhci_hcd_init_usb3_data()  argument
5248 	 * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts  in xhci_hcd_init_usb3_data()
5250 	 * is a two digit BCD containig minor and sub-minor numbers.  in xhci_hcd_init_usb3_data()
5251 	 * This was later clarified in xHCI 1.2.  in xhci_hcd_init_usb3_data()
5256 	if (xhci->usb3_rhub.min_rev == 0x1)  in xhci_hcd_init_usb3_data()
5259 		minor_rev = xhci->usb3_rhub.min_rev / 0x10;  in xhci_hcd_init_usb3_data()
5263 		hcd->speed = HCD_USB32;  in xhci_hcd_init_usb3_data()
5264 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;  in xhci_hcd_init_usb3_data()
5265 		hcd->self.root_hub->rx_lanes = 2;  in xhci_hcd_init_usb3_data()
5266 		hcd->self.root_hub->tx_lanes = 2;  in xhci_hcd_init_usb3_data()
5267 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2;  in xhci_hcd_init_usb3_data()
5270 		hcd->speed = HCD_USB31;  in xhci_hcd_init_usb3_data()
5271 		hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;  in xhci_hcd_init_usb3_data()
5272 		hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1;  in xhci_hcd_init_usb3_data()
5275 	xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",  in xhci_hcd_init_usb3_data()
5278 	xhci->usb3_rhub.hcd = hcd;  in xhci_hcd_init_usb3_data()
5283 	struct xhci_hcd		*xhci;  in xhci_gen_setup()  local
5288 	struct device		*dev = hcd->self.sysdev;  in xhci_gen_setup()
5291 	/* Accept arbitrarily long scatter-gather lists */  in xhci_gen_setup()
5292 	hcd->self.sg_tablesize = ~0;  in xhci_gen_setup()
5295 	hcd->self.no_sg_constraint = 1;  in xhci_gen_setup()
5297 	/* XHCI controllers don't stop the ep queue on short packets :| */  in xhci_gen_setup()
5298 	hcd->self.no_stop_on_short = 1;  in xhci_gen_setup()
5300 	xhci = hcd_to_xhci(hcd);  in xhci_gen_setup()
5303 		xhci_hcd_init_usb3_data(xhci, hcd);  in xhci_gen_setup()
5307 	mutex_init(&xhci->mutex);  in xhci_gen_setup()
5308 	xhci->main_hcd = hcd;  in xhci_gen_setup()
5309 	xhci->cap_regs = hcd->regs;  in xhci_gen_setup()
5310 	xhci->op_regs = hcd->regs +  in xhci_gen_setup()
5311 		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));  in xhci_gen_setup()
5312 	xhci->run_regs = hcd->regs +  in xhci_gen_setup()
5313 		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);  in xhci_gen_setup()
5314 	/* Cache read-only capability registers */  in xhci_gen_setup()
5315 	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);  in xhci_gen_setup()
5316 	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);  in xhci_gen_setup()
5317 	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);  in xhci_gen_setup()
5318 	xhci->hci_version = HC_VERSION(readl(&xhci->cap_regs->hc_capbase));  in xhci_gen_setup()
5319 	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);  in xhci_gen_setup()
5320 	if (xhci->hci_version > 0x100)  in xhci_gen_setup()
5321 		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);  in xhci_gen_setup()
5323 	xhci->quirks |= quirks;  in xhci_gen_setup()
5325 	get_quirks(dev, xhci);  in xhci_gen_setup()
5327 	/* In xhci controllers which follow xhci 1.0 spec gives a spurious  in xhci_gen_setup()
5331 	if (xhci->hci_version > 0x96)  in xhci_gen_setup()
5332 		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;  in xhci_gen_setup()
5335 	retval = xhci_halt(xhci);  in xhci_gen_setup()
5339 	xhci_zero_64b_regs(xhci);  in xhci_gen_setup()
5341 	xhci_dbg(xhci, "Resetting HCD\n");  in xhci_gen_setup()
5343 	retval = xhci_reset(xhci, XHCI_RESET_LONG_USEC);  in xhci_gen_setup()
5346 	xhci_dbg(xhci, "Reset complete\n");  in xhci_gen_setup()
5349 	 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)  in xhci_gen_setup()
5350 	 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit  in xhci_gen_setup()
5352 	 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,  in xhci_gen_setup()
5355 	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)  in xhci_gen_setup()
5356 		xhci->hcc_params &= ~BIT(0);  in xhci_gen_setup()
5358 	/* Set dma_mask and coherent_dma_mask to 64-bits,  in xhci_gen_setup()
5359 	 * if xHC supports 64-bit addressing */  in xhci_gen_setup()
5360 	if (HCC_64BIT_ADDR(xhci->hcc_params) &&  in xhci_gen_setup()
5362 		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");  in xhci_gen_setup()
5366 		 * This is to avoid error in cases where a 32-bit USB  in xhci_gen_setup()
5367 		 * controller is used on a 64-bit capable system.  in xhci_gen_setup()
5372 		xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");  in xhci_gen_setup()
5376 	xhci_dbg(xhci, "Calling HCD init\n");  in xhci_gen_setup()
5381 	xhci_dbg(xhci, "Called HCD init\n");  in xhci_gen_setup()
5384 		xhci_hcd_init_usb3_data(xhci, hcd);  in xhci_gen_setup()
5386 		xhci_hcd_init_usb2_data(xhci, hcd);  in xhci_gen_setup()
5388 	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",  in xhci_gen_setup()
5389 		  xhci->hcc_params, xhci->hci_version, xhci->quirks);  in xhci_gen_setup()
5398 	struct xhci_hcd *xhci;  in xhci_clear_tt_buffer_complete()  local
5404 	xhci = hcd_to_xhci(hcd);  in xhci_clear_tt_buffer_complete()
5406 	spin_lock_irqsave(&xhci->lock, flags);  in xhci_clear_tt_buffer_complete()
5407 	udev = (struct usb_device *)ep->hcpriv;  in xhci_clear_tt_buffer_complete()
5408 	slot_id = udev->slot_id;  in xhci_clear_tt_buffer_complete()
5409 	ep_index = xhci_get_endpoint_index(&ep->desc);  in xhci_clear_tt_buffer_complete()
5411 	xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;  in xhci_clear_tt_buffer_complete()
5412 	xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);  in xhci_clear_tt_buffer_complete()
5413 	spin_unlock_irqrestore(&xhci->lock, flags);  in xhci_clear_tt_buffer_complete()
5417 	.description =		"xhci-hcd",
5418 	.product_desc =		"xHCI Host Controller",
5422 	 * generic hardware linkage
5488 	/* Copy the generic table to drv then apply the overrides */  in xhci_init_driver()
5492 		drv->hcd_priv_size += over->extra_priv_size;  in xhci_init_driver()
5493 		if (over->reset)  in xhci_init_driver()
5494 			drv->reset = over->reset;  in xhci_init_driver()
5495 		if (over->start)  in xhci_init_driver()
5496 			drv->start = over->start;  in xhci_init_driver()
5497 		if (over->add_endpoint)  in xhci_init_driver()
5498 			drv->add_endpoint = over->add_endpoint;  in xhci_init_driver()
5499 		if (over->drop_endpoint)  in xhci_init_driver()
5500 			drv->drop_endpoint = over->drop_endpoint;  in xhci_init_driver()
5501 		if (over->check_bandwidth)  in xhci_init_driver()
5502 			drv->check_bandwidth = over->check_bandwidth;  in xhci_init_driver()
5503 		if (over->reset_bandwidth)  in xhci_init_driver()
5504 			drv->reset_bandwidth = over->reset_bandwidth;  in xhci_init_driver()
5534 		return -ENODEV;  in xhci_hcd_init()