1 /* SPDX-License-Identifier: GPL-2.0 */
4  * xHCI host controller driver
19 #include <linux/io-64-nonatomic-lo-hi.h>
21 /* Code sharing between pci-quirks and xhci hcd */
22 #include	"xhci-ext-caps.h"
23 #include "pci-quirks.h"
28 /* xHCI PCI Configuration Registers */
31 /* Max number of USB devices for any host controller - limit in section 6.1 */
33 /* Section 5.3.3 - MaxPorts */
37  * xHCI register interface.
38  * This corresponds to the eXtensible Host Controller Interface (xHCI)
43  * struct xhci_cap_regs - xHCI Host Controller Capability Registers.
45  * @hcs_params1:	HCSPARAMS1 - Structural Parameters 1
46  * @hcs_params2:	HCSPARAMS2 - Structural Parameters 2
47  * @hcs_params3:	HCSPARAMS3 - Structural Parameters 3
48  * @hcc_params:		HCCPARAMS - Capability Parameters
49  * @db_off:		DBOFF - Doorbell array offset
50  * @run_regs_off:	RTSOFF - Runtime register space offset
51  * @hcc_params2:	HCCPARAMS2 Capability Parameters 2, xhci 1.1 only
61 	__le32	hcc_params2; /* xhci 1.1 */
62 	/* Reserved up to (CAPLENGTH - 0x1C) */
66 /* bits 7:0 - how long is the Capabilities register */
71 /* HCSPARAMS1 - hcs_params1 - bitmasks */
77 /* bits 24:31, Max Ports - max value is 0x7F = 127 ports */
80 /* HCSPARAMS2 - hcs_params2 - bitmasks */
87 /* bit 26 Scratchpad restore - for save/restore HW state - not used yet */
91 /* HCSPARAMS3 - hcs_params3 - bitmasks */
97 /* HCCPARAMS - hcc_params - bitmasks */
98 /* true: HC can use 64-bit address pointers */
102 /* true: HC uses 64-byte Device Context structures
103  * FIXME 64-byte context structures aren't supported yet.
116 /* true: HC supports Stopped - Short Packet */
120 /* Max size for Primary Stream Arrays - 2^(n+1), where n is bits 12:15 */
122 /* Extended Capabilities pointer from PCI base - section 5.3.6 */
127 /* db_off bitmask - bits 0:1 reserved */
130 /* run_regs_off bitmask - bits 0:4 reserved */
133 /* HCCPARAMS2 - hcc_params2 - bitmasks */
158  * struct xhci_op_regs - xHCI Host Controller Operational Registers.
159  * @command:		USBCMD - xHC command register
160  * @status:		USBSTS - xHC status register
165  * @cmd_ring:		CRP - 64-bit Command Ring Pointer
166  * @dcbaa_ptr:		DCBAAP - 64-bit Device Context Base Address Array Pointer
167  * @config_reg:		CONFIG - Configure Register
168  * @port_status_base:	PORTSCn - base address for Port Status and Control
173  * @port_power_base:	PORTPMSCn - base address for
175  * @port_link_base:	PORTLIn - base address for Port Link Info (current
187 	/* rsvd: offset 0x20-2F */
191 	/* rsvd: offset 0x3C-3FF */
198 	/* registers for ports 2-255 */
202 /* USBCMD - USB command - command bitmasks */
203 /* start/stop HC execution - do not write unless HC is halted*/
205 /* Reset HC - resets internal HC state machine and all registers (except
207  * The xHCI driver must reinitialize the xHC after setting this bit.
210 /* Event Interrupt Enable - a '1' allows interrupts from the host controller */
212 /* Host System Error Interrupt Enable - get out-of-band signal for HC errors */
215 /* light reset (port status stays unchanged) - reset completed when this is 0 */
220 /* Enable Wrap Event - '1' means xHC generates an event when MFINDEX wraps. */
222 /* MFINDEX power management - '1' means xHC can stop MFINDEX counter if all root
223  * hubs are in U3 (selective suspend), disconnect, disabled, or powered-off.
225  * disabled, or powered-off state.
232 /* IMAN - Interrupt Management Register */
236 /* USBSTS - USB status - status bitmasks */
237 /* HC not running - set to 1 when run/stop bit is cleared. */
241 /* event interrupt - clear this prior to clearing any IP flags in IR set*/
246 /* save state status - '1' means xHC is saving state */
248 /* restore state status - '1' means xHC is restoring state */
254 /* true: internal Host Controller Error - SW needs to reset and reinitialize */
259  * DNCTRL - Device Notification Control Register - dev_notification bitmasks
270 /* CRCR - Command Ring Control Register - cmd_ring bitmasks */
274 /* stop ring immediately - abort the currently executing command */
279 /* Command Ring pointer - bit mask for the lower 32 bits. */
282 /* CONFIG - Configure Register - config_reg bitmasks */
283 /* bits 0:7 - maximum number of device slots enabled (NumSlotsEn) */
285 /* bit 8: U3 Entry Enabled, assert PLC when root port enters U3, xhci 1.1 */
287 /* bit 9: Configuration Information Enable, xhci 1.1 */
289 /* bits 10:31 - reserved and should be preserved */
291 /* PORTSC - Port Status and Control Register - port_status_base bitmasks */
297 /* true: port has an over-current condition */
301 /* Port Link State - bits 5:8
323  * 0 - undefined speed - port hasn't be initialized by a reset yet
324  * 1 - full speed
325  * 2 - low speed
326  * 3 - high speed
327  * 4 - super speed
328  * 5-15 reserved
356 /* Port Link State Write Strobe - set this when changing link state */
368 /* true: over-current change */
370 /* true: reset change - 1 to 0 transition of PORT_RESET */
372 /* port link status change - set on some port link state transitions:
374  *  ------------------------------------------------------------------------------
375  *  - U3 to Resume			Wakeup signaling from a device
376  *  - Resume to Recovery to U0		USB 3.0 device resume
377  *  - Resume to U0			USB 2.0 device resume
378  *  - U3 to Recovery to U0		Software resume of USB 3.0 device complete
379  *  - U3 to U0				Software resume of USB 2.0 device complete
380  *  - U2 to U0				L1 resume of USB 2.1 device complete
381  *  - U0 to U0 (???)			L1 entry rejection by USB 2.1 device
382  *  - U0 to disabled			L1 entry error with USB 2.1 device
383  *  - Any state to inactive		Error on USB 3.0 port
386 /* port configure error change - port failed to configure its link partner */
392 /* Cold Attach Status - xHC can set this bit to report device attached during
401 /* wake on over-current (enable) */
404 /* true: device is non-removable - for USB 3.0 roothub emulation */
406 /* Initiate a warm port reset - complete when PORT_WRC is '1' */
409 /* We mark duplicate entries with -1 */
410 #define DUPLICATE_ENTRY ((u8)(-1))
412 /* Port Power Management Status and Control - port_power_base bitmasks */
450  * XHCI 1.0 errata 8/14/12 Table 13 notes:
467  * struct xhci_intr_reg - Interrupt Register Set
468  * @irq_pending:	IMAN - Interrupt Management Register.  Used to enable
470  * @irq_control:	IMOD - Interrupt Moderation Register.
476  * Each interrupter (defined by a MSI-X vector) has an event ring and an Event
495 /* THIS IS BUGGY - FIXME - IP IS WRITE 1 TO CLEAR */
506 /* Counter used to count down the time to the next interrupt - HW use only */
514 /* Dequeue ERST Segment Index (DESI) - Segment number (or alias)
518 /* Event Handler Busy (EHB) - is the event ring scheduled to be serviced by
527  * 		MFINDEX - current microframe number
542  * Bits  0 -  7: Endpoint target
543  * Bits  8 - 15: RsvdZ
544  * Bits 16 - 31: Stream ID
561  * @port_info:		Port offset, count, and protocol-defined information.
615  * Slot Context - section 6.2.1.1.  This assumes the HC uses 32-byte context
616  * structures.  If the HC uses 64-byte contexts, there is an additional 32 bytes
629 /* Route String - 0:19 */
631 /* Device speed - values defined by PORTSC Device Speed field - 20:23 */
635 /* Is this LS/FS device connected through a HS hub? - bit 25 */
637 /* Set if the device is a hub - bit 26 */
639 /* Index of the last valid endpoint context in this device context - 27:31 */
642 #define LAST_CTX_TO_EP_NUM(p)	(((p) >> 27) - 1)
647 /* Max Exit Latency (ms) - worst case time to wake up all links in dev path */
658  * TT Hub Slot ID - for low or full speed devices attached to a high-speed hub
660  * this low or full-speed device.  '0' if attached to root hub port.
664  * The number of the downstream facing port of the high-speed hub
672 /* USB device address - assigned by the HC */
691  * @deq:	64-bit ring dequeue pointer address.  If the endpoint only
699  * Endpoint Context - section 6.2.1.2.  This assumes the HC uses 32-byte context
700  * structures.  If the HC uses 64-byte contexts, there is an additional 32 bytes
708 	/* offset 0x14 - 0x1f reserved for HC internal use */
714  * Endpoint State - bits 0:2
715  * 0 - disabled
716  * 1 - running
717  * 2 - halted due to halt condition - ok to manipulate endpoint ring
718  * 3 - stopped
719  * 4 - TRB error
720  * 5-7 - reserved
728 #define GET_EP_CTX_STATE(ctx)	(le32_to_cpu((ctx)->ep_info) & EP_STATE_MASK)
730 /* Mult - Max number of burtst within an interval, in EP companion desc. */
735 /* Interval - period between requests to an endpoint - 125u increments. */
749  * Force Event - generate transfer events for all TRBs for this endpoint
764 /* bit 7 is Host Initiate Disable - for disabling stream selection */
796 	(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))
798 	(le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1)))
801  * It's useful to pre-allocate these for commands that cannot fail due to
802  * out-of-memory errors, like freeing streams.
823 	/* 64-bit stream ring address, cycle state, and stream type */
825 	/* offset 0x14 - 0x1f reserved for HC internal use */
829 /* Stream Context Types (section 6.4.1) - bits 3:1 of stream ctx deq ptr */
862 /* Some Intel xHCI host controllers need software to keep track of the bus
865  * multi-TT hub) as a separate bandwidth domain.  The direct memory interface
869 	/* ep_interval is zero-based */
871 	/* mult and num_packets are one-based */
918 /* Percentage of bus bandwidth reserved for non-periodic transfers */
946 	/* ----  Related to URB cancellation ---- */
950 	struct xhci_hcd		*xhci;  member
1026 	 * See xhci 1.1 section 4.8.3 for more details
1062  * @dev_context_ptr	array of 64-bit DMA addresses for device contexts
1065 	/* 64-bit device addresses; we only write 32-bit addresses */
1070 /* TODO: write function to set the 64-bit device DMA address */
1078 	/* 64-bit buffer address, or immediate data */
1092 /* Completion Code - only applicable for some types of TRBs */
1190 		return "Stopped - Length Invalid";  in xhci_trb_comp_code_string()
1192 		return "Stopped - Short Packet";  in xhci_trb_comp_code_string()
1213 	/* 64-bit segment pointer*/
1232 /* Address device - disable SetAddress */
1235 /* Configure Endpoint - Deconfigure */
1238 /* Stop Ring - Transfer State Preserve */
1270 /* Stop Endpoint TRB - ep_index to endpoint ID for this TRB */
1271 #define TRB_TO_EP_INDEX(p)		((((p) & (0x1f << 16)) >> 16) - 1)
1287 /* Port ID - bits 31:24 */
1293 /* transfer_len bitmasks - bits 0:16 */
1298 /* xhci 1.1 uses the TD_SIZE field for TBC if Extended TBC is enabled (ETE) */
1300 /* Interrupter Target - which MSI-X vector to target the completion event at */
1303 /* Total burst count field, Rsvdz on xhci 1.1 with Extended TBC enabled (ETE) */
1307 /* Cycle bit - indicates TRB ownership by HC or HCD */
1352 	struct xhci_generic_trb		generic;  member
1373 /* Transfer Ring No-op (not for the command ring) */
1402 /* Force Header Command - generate a transaction or link management packet */
1404 /* No-op Command - not for transfer rings */
1406 /* TRB IDs 24-31 reserved */
1420 /* Device Notification Event - device sent function wake notification */
1422 /* MFINDEX Wrap Event - microframe counter wrapped */
1424 /* TRB IDs 40-47 reserved, 48-63 is vendor-defined */
1426 /* Nec vendor-specific command completion event. */
1449 		return "No-Op";  in xhci_trb_type_string()
1479 		return "No-Op Command";  in xhci_trb_type_string()
1506 /* Above, but for __le32 types -- can avoid work by swapping constants: */
1517  * since the command ring is 64-byte aligned.
1522 #define MAX_RSVD_CMD_TRBS	(TRBS_PER_SEGMENT - 3)
1529 #define TRB_BUFF_LEN_UP_TO_BOUNDARY(addr)	(TRB_MAX_BUFF_SIZE - \
1530 					(addr & (TRB_MAX_BUFF_SIZE - 1)))
1544 	/* Max packet sized bounce buffer for td-fragmant alignment */
1574 /* xHCI command default timeout value */
1641 	/* 64-bit event ring segment address */
1651 	/* xhci->event_ring keeps track of segment dma addresses */
1704 	/* ports suspend status arrays - max 31 ports for USB2, 15 for USB3 */
1720  * Intel Lynx Point LP xHCI host.
1762 	/* Cached register copies of read-only HC data */
1786 	/* msi-x vectors */
1833 /* Host controller is dying - not responding to commands. "I'm not dead yet!"
1836  * halt the xHCI host, and complete all URBs with an -ESHUTDOWN code.  Any code
1839  * they see this status (any time they drop and re-acquire xhci->lock).
1843  * There are no reports of xHCI host controllers that display this issue.
1909 	/* support xHCI 1.0 spec USB2 hardware LPM */
1931 	/* platform-specific data -- must come last */
1935 /* Platform specific overrides to generic XHCI hc_driver ops */
1958 		primary_hcd = hcd->primary_hcd;  in hcd_to_xhci()
1960 	return (struct xhci_hcd *) (primary_hcd->hcd_priv);  in hcd_to_xhci()
1963 static inline struct usb_hcd *xhci_to_hcd(struct xhci_hcd *xhci)  in xhci_to_hcd()  argument
1965 	return xhci->main_hcd;  in xhci_to_hcd()
1968 #define xhci_dbg(xhci, fmt, args...) \  argument
1969 	dev_dbg(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1970 #define xhci_err(xhci, fmt, args...) \  argument
1971 	dev_err(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1972 #define xhci_warn(xhci, fmt, args...) \  argument
1973 	dev_warn(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1974 #define xhci_warn_ratelimited(xhci, fmt, args...) \  argument
1975 	dev_warn_ratelimited(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1976 #define xhci_info(xhci, fmt, args...) \  argument
1977 	dev_info(xhci_to_hcd(xhci)->self.controller , fmt , ## args)
1982  * Some xHCI implementations may support 64-bit address pointers.  Registers
1983  * with 64-bit address pointers should be written to with dword accesses by
1985  * xHCI implementations that do not support 64-bit address pointers will ignore
1988 static inline u64 xhci_read_64(const struct xhci_hcd *xhci,  in xhci_read_64()  argument
1993 static inline void xhci_write_64(struct xhci_hcd *xhci,  in xhci_write_64()  argument
1999 static inline int xhci_link_trb_quirk(struct xhci_hcd *xhci)  in xhci_link_trb_quirk()  argument
2001 	return xhci->quirks & XHCI_LINK_TRB_QUIRK;  in xhci_link_trb_quirk()
2004 /* xHCI debugging */
2005 char *xhci_get_slot_state(struct xhci_hcd *xhci,
2007 void xhci_dbg_trace(struct xhci_hcd *xhci, void (*trace)(struct va_format *),
2010 /* xHCI memory management */
2011 void xhci_mem_cleanup(struct xhci_hcd *xhci);
2012 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags);
2013 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id);
2014 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, struct usb_device *udev, gfp_t flags…
2015 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev);
2016 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
2021 void xhci_endpoint_zero(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev, struct usb_host_e…
2022 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2026 void xhci_update_bw_info(struct xhci_hcd *xhci,
2030 void xhci_endpoint_copy(struct xhci_hcd *xhci,
2034 void xhci_slot_copy(struct xhci_hcd *xhci,
2037 int xhci_endpoint_init(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev,
2040 struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
2043 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring);
2044 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,
2046 int xhci_alloc_erst(struct xhci_hcd *xhci,
2052 void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst);
2053 void xhci_free_endpoint_ring(struct xhci_hcd *xhci,
2056 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
2060 void xhci_free_stream_info(struct xhci_hcd *xhci,
2062 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
2067 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
2072 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
2074 struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci,
2077 void xhci_free_command(struct xhci_hcd *xhci,
2079 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
2081 void xhci_free_container_ctx(struct xhci_hcd *xhci,
2084 /* xHCI host controller glue */
2087 void xhci_quiesce(struct xhci_hcd *xhci);
2088 int xhci_halt(struct xhci_hcd *xhci);
2089 int xhci_start(struct xhci_hcd *xhci);
2090 int xhci_reset(struct xhci_hcd *xhci);
2102 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id);
2103 int xhci_ext_cap_init(struct xhci_hcd *xhci);
2105 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup);
2106 int xhci_resume(struct xhci_hcd *xhci, bool hibernated);
2111 int xhci_alloc_tt_info(struct xhci_hcd *xhci,
2116 /* xHCI ring, segment, TRB, and TD functions */
2118 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
2121 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code);
2122 void xhci_ring_cmd_db(struct xhci_hcd *xhci);
2123 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
2125 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
2127 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
2129 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
2131 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
2133 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
2135 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, struct urb *urb,
2137 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
2139 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
2142 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
2144 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
2147 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
2149 void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci, unsigned int slot_id,
2155 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, unsigned int slot_id,
2157 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
2160 void xhci_cleanup_command_queue(struct xhci_hcd *xhci);
2161 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring);
2164 /* xHCI roothub code */
2165 void xhci_set_link_state(struct xhci_hcd *xhci, struct xhci_port *port,
2167 void xhci_test_and_clear_bit(struct xhci_hcd *xhci, struct xhci_port *port,
2175 void xhci_hc_died(struct xhci_hcd *xhci);
2188 int xhci_find_slot_id_by_port(struct usb_hcd *hcd, struct xhci_hcd *xhci,
2190 void xhci_ring_device(struct xhci_hcd *xhci, int slot_id);
2192 /* xHCI contexts */
2194 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx);
2195 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, struct xhci_container_ctx *ctx, unsigned…
2197 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
2201 static inline struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,  in xhci_urb_to_transfer_ring()  argument
2204 	return xhci_triad_to_transfer_ring(xhci, urb->dev->slot_id,  in xhci_urb_to_transfer_ring()
2205 					xhci_get_endpoint_index(&urb->ep->desc),  in xhci_urb_to_transfer_ring()
2206 					urb->stream_id);  in xhci_urb_to_transfer_ring()
2216 	if (!usb_endpoint_xfer_isoc(&urb->ep->desc) && usb_urb_dir_out(urb) &&  in xhci_urb_suitable_for_idt()
2217 	    usb_endpoint_maxp(&urb->ep->desc) >= TRB_IDT_MAX_SIZE &&  in xhci_urb_suitable_for_idt()
2218 	    urb->transfer_buffer_length <= TRB_IDT_MAX_SIZE &&  in xhci_urb_suitable_for_idt()
2219 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) &&  in xhci_urb_suitable_for_idt()
2220 	    !urb->num_sgs)  in xhci_urb_suitable_for_idt()
2456 			"type '%s' -> raw %08x %08x %08x %08x",  in xhci_decode_trb()
2509 				s = "full-speed";  in xhci_decode_slot_context()
2512 				s = "low-speed";  in xhci_decode_slot_context()
2515 				s = "high-speed";  in xhci_decode_slot_context()
2518 				s = "super-speed";  in xhci_decode_slot_context()
2521 				s = "super-speed plus";  in xhci_decode_slot_context()
2526 			mtt ? " multi-TT" : "",  in xhci_decode_slot_context()
2583 		      portsc & PORT_POWER	? "Powered" : "Powered-off",  in xhci_decode_portsc()
2584 		      portsc & PORT_CONNECT	? "Connected" : "Not-connected",  in xhci_decode_portsc()
2592 		ret += sprintf(str + ret, "In-Reset ");  in xhci_decode_portsc()