Lines Matching refs:xhci
28 static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, in xhci_segment_alloc() argument
36 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_segment_alloc()
42 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); in xhci_segment_alloc()
52 dma_pool_free(xhci->segment_pool, seg->trbs, dma); in xhci_segment_alloc()
68 static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) in xhci_segment_free() argument
71 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); in xhci_segment_free()
78 static void xhci_free_segments_for_ring(struct xhci_hcd *xhci, in xhci_free_segments_for_ring() argument
86 xhci_segment_free(xhci, seg); in xhci_free_segments_for_ring()
89 xhci_segment_free(xhci, first); in xhci_free_segments_for_ring()
126 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_link_rings() argument
137 chain_links = !!(xhci_link_trb_quirk(xhci) || in xhci_link_rings()
139 (xhci->quirks & XHCI_AMD_0x96_HOST))); in xhci_link_rings()
280 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_ring_free() argument
290 xhci_free_segments_for_ring(xhci, ring->first_seg); in xhci_ring_free()
321 static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, in xhci_alloc_segments_for_ring() argument
330 chain_links = !!(xhci_link_trb_quirk(xhci) || in xhci_alloc_segments_for_ring()
332 (xhci->quirks & XHCI_AMD_0x96_HOST))); in xhci_alloc_segments_for_ring()
334 prev = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
343 next = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
348 xhci_segment_free(xhci, prev); in xhci_alloc_segments_for_ring()
371 struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, in xhci_ring_alloc() argument
377 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_ring_alloc()
390 ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, in xhci_ring_alloc()
411 void xhci_free_endpoint_ring(struct xhci_hcd *xhci, in xhci_free_endpoint_ring() argument
415 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); in xhci_free_endpoint_ring()
423 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion() argument
439 ret = xhci_alloc_segments_for_ring(xhci, &first, &last, in xhci_ring_expansion()
452 xhci_segment_free(xhci, first); in xhci_ring_expansion()
460 xhci_link_rings(xhci, ring, first, last, num_segs); in xhci_ring_expansion()
462 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in xhci_ring_expansion()
469 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, in xhci_alloc_container_ctx() argument
473 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_container_ctx()
483 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; in xhci_alloc_container_ctx()
485 ctx->size += CTX_SIZE(xhci->hcc_params); in xhci_alloc_container_ctx()
487 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); in xhci_alloc_container_ctx()
495 void xhci_free_container_ctx(struct xhci_hcd *xhci, in xhci_free_container_ctx() argument
500 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma); in xhci_free_container_ctx()
513 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, in xhci_get_slot_ctx() argument
520 (ctx->bytes + CTX_SIZE(xhci->hcc_params)); in xhci_get_slot_ctx()
523 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, in xhci_get_ep_ctx() argument
533 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); in xhci_get_ep_ctx()
539 static void xhci_free_stream_ctx(struct xhci_hcd *xhci, in xhci_free_stream_ctx() argument
543 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_stream_ctx()
550 return dma_pool_free(xhci->small_streams_pool, in xhci_free_stream_ctx()
553 return dma_pool_free(xhci->medium_streams_pool, in xhci_free_stream_ctx()
567 static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, in xhci_alloc_stream_ctx() argument
571 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_ctx()
578 return dma_pool_alloc(xhci->small_streams_pool, in xhci_alloc_stream_ctx()
581 return dma_pool_alloc(xhci->medium_streams_pool, in xhci_alloc_stream_ctx()
621 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, in xhci_alloc_stream_info() argument
631 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_info()
633 xhci_dbg(xhci, "Allocating %u streams and %u " in xhci_alloc_stream_info()
636 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { in xhci_alloc_stream_info()
637 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); in xhci_alloc_stream_info()
640 xhci->cmd_ring_reserved_trbs++; in xhci_alloc_stream_info()
658 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci, in xhci_alloc_stream_info()
668 xhci_alloc_command_with_ctx(xhci, true, mem_flags); in xhci_alloc_stream_info()
681 xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet, in xhci_alloc_stream_info()
694 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", in xhci_alloc_stream_info()
699 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
717 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
721 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_alloc_stream_info()
727 xhci->cmd_ring_reserved_trbs--; in xhci_alloc_stream_info()
734 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, in xhci_setup_streams_ep_input_ctx() argument
744 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, in xhci_setup_streams_ep_input_ctx()
771 void xhci_free_stream_info(struct xhci_hcd *xhci, in xhci_free_stream_info() argument
784 xhci_ring_free(xhci, cur_ring); in xhci_free_stream_info()
788 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_free_stream_info()
789 xhci->cmd_ring_reserved_trbs--; in xhci_free_stream_info()
791 xhci_free_stream_ctx(xhci, in xhci_free_stream_info()
803 static void xhci_init_endpoint_timer(struct xhci_hcd *xhci, in xhci_init_endpoint_timer() argument
808 ep->xhci = xhci; in xhci_init_endpoint_timer()
811 static void xhci_free_tt_info(struct xhci_hcd *xhci, in xhci_free_tt_info() argument
823 virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_tt_info()
824 xhci_dbg(xhci, "Bad real port.\n"); in xhci_free_tt_info()
828 tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_free_tt_info()
841 int xhci_alloc_tt_info(struct xhci_hcd *xhci, in xhci_alloc_tt_info() argument
849 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_tt_info()
865 &xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_alloc_tt_info()
876 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); in xhci_alloc_tt_info()
886 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_device() argument
893 if (slot_id == 0 || !xhci->devs[slot_id]) in xhci_free_virt_device()
896 dev = xhci->devs[slot_id]; in xhci_free_virt_device()
898 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; in xhci_free_virt_device()
909 xhci_ring_free(xhci, dev->eps[i].ring); in xhci_free_virt_device()
911 xhci_free_stream_info(xhci, in xhci_free_virt_device()
919 xhci_warn(xhci, "Slot %u endpoint %u " in xhci_free_virt_device()
924 xhci_free_tt_info(xhci, dev, slot_id); in xhci_free_virt_device()
926 xhci_update_tt_active_eps(xhci, dev, old_active_eps); in xhci_free_virt_device()
929 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_free_virt_device()
931 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_free_virt_device()
935 kfree(xhci->devs[slot_id]); in xhci_free_virt_device()
936 xhci->devs[slot_id] = NULL; in xhci_free_virt_device()
945 static void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_devices_depth_first() argument
952 vdev = xhci->devs[slot_id]; in xhci_free_virt_devices_depth_first()
957 vdev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_virt_devices_depth_first()
958 xhci_dbg(xhci, "Bad vdev->real_port.\n"); in xhci_free_virt_devices_depth_first()
962 tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts); in xhci_free_virt_devices_depth_first()
967 for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_free_virt_devices_depth_first()
968 vdev = xhci->devs[i]; in xhci_free_virt_devices_depth_first()
971 xhci, i); in xhci_free_virt_devices_depth_first()
977 xhci_debugfs_remove_slot(xhci, slot_id); in xhci_free_virt_devices_depth_first()
978 xhci_free_virt_device(xhci, slot_id); in xhci_free_virt_devices_depth_first()
981 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, in xhci_alloc_virt_device() argument
988 if (slot_id == 0 || xhci->devs[slot_id]) { in xhci_alloc_virt_device()
989 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id); in xhci_alloc_virt_device()
998 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); in xhci_alloc_virt_device()
1002 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
1006 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); in xhci_alloc_virt_device()
1010 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
1015 xhci_init_endpoint_timer(xhci, &dev->eps[i]); in xhci_alloc_virt_device()
1021 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags); in xhci_alloc_virt_device()
1028 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); in xhci_alloc_virt_device()
1029 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", in xhci_alloc_virt_device()
1031 &xhci->dcbaa->dev_context_ptrs[slot_id], in xhci_alloc_virt_device()
1032 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); in xhci_alloc_virt_device()
1036 xhci->devs[slot_id] = dev; in xhci_alloc_virt_device()
1042 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_alloc_virt_device()
1044 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_alloc_virt_device()
1050 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, in xhci_copy_ep0_dequeue_into_input_ctx() argument
1057 virt_dev = xhci->devs[udev->slot_id]; in xhci_copy_ep0_dequeue_into_input_ctx()
1058 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0); in xhci_copy_ep0_dequeue_into_input_ctx()
1082 static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, in xhci_find_real_port_number() argument
1089 hcd = xhci->shared_hcd; in xhci_find_real_port_number()
1091 hcd = xhci->main_hcd; in xhci_find_real_port_number()
1101 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev) in xhci_setup_addressable_virt_dev() argument
1110 dev = xhci->devs[udev->slot_id]; in xhci_setup_addressable_virt_dev()
1113 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n", in xhci_setup_addressable_virt_dev()
1117 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); in xhci_setup_addressable_virt_dev()
1118 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); in xhci_setup_addressable_virt_dev()
1145 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); in xhci_setup_addressable_virt_dev()
1153 port_num = xhci_find_real_port_number(xhci, udev); in xhci_setup_addressable_virt_dev()
1163 xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num); in xhci_setup_addressable_virt_dev()
1164 xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port); in xhci_setup_addressable_virt_dev()
1173 dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table; in xhci_setup_addressable_virt_dev()
1178 rh_bw = &xhci->rh_bw[port_num - 1]; in xhci_setup_addressable_virt_dev()
1193 xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); in xhci_setup_addressable_virt_dev()
1203 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); in xhci_setup_addressable_virt_dev()
1204 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); in xhci_setup_addressable_virt_dev()
1430 int xhci_endpoint_init(struct xhci_hcd *xhci, in xhci_endpoint_init() argument
1450 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_init()
1470 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && in xhci_endpoint_init()
1497 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) in xhci_endpoint_init()
1500 if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2)) in xhci_endpoint_init()
1505 xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags); in xhci_endpoint_init()
1529 void xhci_endpoint_zero(struct xhci_hcd *xhci, in xhci_endpoint_zero() argument
1537 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_zero()
1558 void xhci_update_bw_info(struct xhci_hcd *xhci, in xhci_update_bw_info() argument
1583 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); in xhci_update_bw_info()
1616 void xhci_endpoint_copy(struct xhci_hcd *xhci, in xhci_endpoint_copy() argument
1624 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); in xhci_endpoint_copy()
1625 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index); in xhci_endpoint_copy()
1631 if (xhci->quirks & XHCI_MTK_HOST) { in xhci_endpoint_copy()
1642 void xhci_slot_copy(struct xhci_hcd *xhci, in xhci_slot_copy() argument
1649 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx); in xhci_slot_copy()
1650 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx); in xhci_slot_copy()
1659 static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) in scratchpad_alloc() argument
1662 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_alloc()
1663 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_alloc()
1665 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in scratchpad_alloc()
1671 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, in scratchpad_alloc()
1673 if (!xhci->scratchpad) in scratchpad_alloc()
1676 xhci->scratchpad->sp_array = dma_alloc_coherent(dev, in scratchpad_alloc()
1678 &xhci->scratchpad->sp_dma, flags); in scratchpad_alloc()
1679 if (!xhci->scratchpad->sp_array) in scratchpad_alloc()
1682 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), in scratchpad_alloc()
1684 if (!xhci->scratchpad->sp_buffers) in scratchpad_alloc()
1687 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); in scratchpad_alloc()
1690 void *buf = dma_alloc_coherent(dev, xhci->page_size, &dma, in scratchpad_alloc()
1695 xhci->scratchpad->sp_array[i] = dma; in scratchpad_alloc()
1696 xhci->scratchpad->sp_buffers[i] = buf; in scratchpad_alloc()
1703 dma_free_coherent(dev, xhci->page_size, in scratchpad_alloc()
1704 xhci->scratchpad->sp_buffers[i], in scratchpad_alloc()
1705 xhci->scratchpad->sp_array[i]); in scratchpad_alloc()
1708 kfree(xhci->scratchpad->sp_buffers); in scratchpad_alloc()
1712 xhci->scratchpad->sp_array, in scratchpad_alloc()
1713 xhci->scratchpad->sp_dma); in scratchpad_alloc()
1716 kfree(xhci->scratchpad); in scratchpad_alloc()
1717 xhci->scratchpad = NULL; in scratchpad_alloc()
1723 static void scratchpad_free(struct xhci_hcd *xhci) in scratchpad_free() argument
1727 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_free()
1729 if (!xhci->scratchpad) in scratchpad_free()
1732 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_free()
1735 dma_free_coherent(dev, xhci->page_size, in scratchpad_free()
1736 xhci->scratchpad->sp_buffers[i], in scratchpad_free()
1737 xhci->scratchpad->sp_array[i]); in scratchpad_free()
1739 kfree(xhci->scratchpad->sp_buffers); in scratchpad_free()
1741 xhci->scratchpad->sp_array, in scratchpad_free()
1742 xhci->scratchpad->sp_dma); in scratchpad_free()
1743 kfree(xhci->scratchpad); in scratchpad_free()
1744 xhci->scratchpad = NULL; in scratchpad_free()
1747 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, in xhci_alloc_command() argument
1751 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_command()
1773 struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, in xhci_alloc_command_with_ctx() argument
1778 command = xhci_alloc_command(xhci, allocate_completion, mem_flags); in xhci_alloc_command_with_ctx()
1782 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, in xhci_alloc_command_with_ctx()
1797 void xhci_free_command(struct xhci_hcd *xhci, in xhci_free_command() argument
1800 xhci_free_container_ctx(xhci, in xhci_free_command()
1806 int xhci_alloc_erst(struct xhci_hcd *xhci, in xhci_alloc_erst() argument
1817 erst->entries = dma_alloc_coherent(xhci_to_hcd(xhci)->self.sysdev, in xhci_alloc_erst()
1836 void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) in xhci_free_erst() argument
1839 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_erst()
1849 void xhci_mem_cleanup(struct xhci_hcd *xhci) in xhci_mem_cleanup() argument
1851 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_cleanup()
1854 cancel_delayed_work_sync(&xhci->cmd_timer); in xhci_mem_cleanup()
1856 xhci_free_erst(xhci, &xhci->erst); in xhci_mem_cleanup()
1858 if (xhci->event_ring) in xhci_mem_cleanup()
1859 xhci_ring_free(xhci, xhci->event_ring); in xhci_mem_cleanup()
1860 xhci->event_ring = NULL; in xhci_mem_cleanup()
1861 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); in xhci_mem_cleanup()
1863 if (xhci->lpm_command) in xhci_mem_cleanup()
1864 xhci_free_command(xhci, xhci->lpm_command); in xhci_mem_cleanup()
1865 xhci->lpm_command = NULL; in xhci_mem_cleanup()
1866 if (xhci->cmd_ring) in xhci_mem_cleanup()
1867 xhci_ring_free(xhci, xhci->cmd_ring); in xhci_mem_cleanup()
1868 xhci->cmd_ring = NULL; in xhci_mem_cleanup()
1869 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); in xhci_mem_cleanup()
1870 xhci_cleanup_command_queue(xhci); in xhci_mem_cleanup()
1872 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_mem_cleanup()
1873 for (i = 0; i < num_ports && xhci->rh_bw; i++) { in xhci_mem_cleanup()
1874 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; in xhci_mem_cleanup()
1882 for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) in xhci_mem_cleanup()
1883 xhci_free_virt_devices_depth_first(xhci, i); in xhci_mem_cleanup()
1885 dma_pool_destroy(xhci->segment_pool); in xhci_mem_cleanup()
1886 xhci->segment_pool = NULL; in xhci_mem_cleanup()
1887 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); in xhci_mem_cleanup()
1889 dma_pool_destroy(xhci->device_pool); in xhci_mem_cleanup()
1890 xhci->device_pool = NULL; in xhci_mem_cleanup()
1891 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); in xhci_mem_cleanup()
1893 dma_pool_destroy(xhci->small_streams_pool); in xhci_mem_cleanup()
1894 xhci->small_streams_pool = NULL; in xhci_mem_cleanup()
1895 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1898 dma_pool_destroy(xhci->medium_streams_pool); in xhci_mem_cleanup()
1899 xhci->medium_streams_pool = NULL; in xhci_mem_cleanup()
1900 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1903 if (xhci->dcbaa) in xhci_mem_cleanup()
1904 dma_free_coherent(dev, sizeof(*xhci->dcbaa), in xhci_mem_cleanup()
1905 xhci->dcbaa, xhci->dcbaa->dma); in xhci_mem_cleanup()
1906 xhci->dcbaa = NULL; in xhci_mem_cleanup()
1908 scratchpad_free(xhci); in xhci_mem_cleanup()
1910 if (!xhci->rh_bw) in xhci_mem_cleanup()
1915 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { in xhci_mem_cleanup()
1922 xhci->cmd_ring_reserved_trbs = 0; in xhci_mem_cleanup()
1923 xhci->usb2_rhub.num_ports = 0; in xhci_mem_cleanup()
1924 xhci->usb3_rhub.num_ports = 0; in xhci_mem_cleanup()
1925 xhci->num_active_eps = 0; in xhci_mem_cleanup()
1926 kfree(xhci->usb2_rhub.ports); in xhci_mem_cleanup()
1927 kfree(xhci->usb3_rhub.ports); in xhci_mem_cleanup()
1928 kfree(xhci->hw_ports); in xhci_mem_cleanup()
1929 kfree(xhci->rh_bw); in xhci_mem_cleanup()
1930 kfree(xhci->ext_caps); in xhci_mem_cleanup()
1931 for (i = 0; i < xhci->num_port_caps; i++) in xhci_mem_cleanup()
1932 kfree(xhci->port_caps[i].psi); in xhci_mem_cleanup()
1933 kfree(xhci->port_caps); in xhci_mem_cleanup()
1934 xhci->num_port_caps = 0; in xhci_mem_cleanup()
1936 xhci->usb2_rhub.ports = NULL; in xhci_mem_cleanup()
1937 xhci->usb3_rhub.ports = NULL; in xhci_mem_cleanup()
1938 xhci->hw_ports = NULL; in xhci_mem_cleanup()
1939 xhci->rh_bw = NULL; in xhci_mem_cleanup()
1940 xhci->ext_caps = NULL; in xhci_mem_cleanup()
1942 xhci->page_size = 0; in xhci_mem_cleanup()
1943 xhci->page_shift = 0; in xhci_mem_cleanup()
1944 xhci->usb2_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
1945 xhci->usb3_rhub.bus_state.bus_suspended = 0; in xhci_mem_cleanup()
1948 static int xhci_test_trb_in_td(struct xhci_hcd *xhci, in xhci_test_trb_in_td() argument
1963 seg = trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, false); in xhci_test_trb_in_td()
1965 xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n", in xhci_test_trb_in_td()
1967 xhci_warn(xhci, "Tested TRB math w/ seg %p and " in xhci_test_trb_in_td()
1971 xhci_warn(xhci, "starting TRB %p (0x%llx DMA), " in xhci_test_trb_in_td()
1975 xhci_warn(xhci, "Expected seg %p, got seg %p\n", in xhci_test_trb_in_td()
1977 trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, in xhci_test_trb_in_td()
1985 static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) in xhci_check_trb_in_td_math() argument
1994 { xhci->event_ring->first_seg->dma - 16, NULL }, in xhci_check_trb_in_td_math()
1996 { xhci->event_ring->first_seg->dma - 1, NULL }, in xhci_check_trb_in_td_math()
1998 { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
2000 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16, in xhci_check_trb_in_td_math()
2001 xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
2003 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL }, in xhci_check_trb_in_td_math()
2005 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL }, in xhci_check_trb_in_td_math()
2017 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2018 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2019 .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2020 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2024 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2025 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2026 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2027 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2031 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2032 .start_trb = xhci->cmd_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2033 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2034 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2038 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2039 .start_trb = &xhci->event_ring->first_seg->trbs[0], in xhci_check_trb_in_td_math()
2040 .end_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2041 .input_dma = xhci->event_ring->first_seg->dma + 4*16, in xhci_check_trb_in_td_math()
2045 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2046 .start_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2047 .end_trb = &xhci->event_ring->first_seg->trbs[6], in xhci_check_trb_in_td_math()
2048 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2052 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2053 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2054 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2055 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2059 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2060 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2061 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2062 .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16, in xhci_check_trb_in_td_math()
2066 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2067 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2068 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2069 .input_dma = xhci->cmd_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2079 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2080 xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2081 xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2082 &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2092 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2102 xhci_dbg(xhci, "TRB math tests passed.\n"); in xhci_check_trb_in_td_math()
2106 static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) in xhci_set_hc_event_deq() argument
2111 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_set_hc_event_deq()
2112 xhci->event_ring->dequeue); in xhci_set_hc_event_deq()
2114 xhci_warn(xhci, "WARN something wrong with SW event ring " in xhci_set_hc_event_deq()
2117 temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2123 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_set_hc_event_deq()
2126 xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, in xhci_set_hc_event_deq()
2127 &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2130 static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, in xhci_add_in_port() argument
2137 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_add_in_port()
2145 rhub = &xhci->usb3_rhub; in xhci_add_in_port()
2147 rhub = &xhci->usb2_rhub; in xhci_add_in_port()
2149 xhci_warn(xhci, "Ignoring unknown port speed, " in xhci_add_in_port()
2164 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2173 port_cap = &xhci->port_caps[xhci->num_port_caps++]; in xhci_add_in_port()
2174 if (xhci->num_port_caps > max_caps) in xhci_add_in_port()
2199 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", in xhci_add_in_port()
2209 if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) in xhci_add_in_port()
2210 xhci->ext_caps[xhci->num_ext_caps++] = temp; in xhci_add_in_port()
2212 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03) && in xhci_add_in_port()
2214 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2216 xhci->hw_lpm_support = 1; in xhci_add_in_port()
2221 struct xhci_port *hw_port = &xhci->hw_ports[i]; in xhci_add_in_port()
2224 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p," in xhci_add_in_port()
2226 xhci_warn(xhci, "Port was marked as USB %u, " in xhci_add_in_port()
2246 static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, in xhci_create_rhub_port_array() argument
2251 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_create_rhub_port_array()
2257 for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { in xhci_create_rhub_port_array()
2258 if (xhci->hw_ports[i].rhub != rhub || in xhci_create_rhub_port_array()
2259 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) in xhci_create_rhub_port_array()
2261 xhci->hw_ports[i].hcd_portnum = port_index; in xhci_create_rhub_port_array()
2262 rhub->ports[port_index] = &xhci->hw_ports[i]; in xhci_create_rhub_port_array()
2276 static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) in xhci_setup_port_arrays() argument
2284 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_setup_port_arrays()
2286 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_setup_port_arrays()
2287 xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports), in xhci_setup_port_arrays()
2289 if (!xhci->hw_ports) in xhci_setup_port_arrays()
2293 xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base + in xhci_setup_port_arrays()
2295 xhci->hw_ports[i].hw_portnum = i; in xhci_setup_port_arrays()
2298 xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags, in xhci_setup_port_arrays()
2300 if (!xhci->rh_bw) in xhci_setup_port_arrays()
2305 INIT_LIST_HEAD(&xhci->rh_bw[i].tts); in xhci_setup_port_arrays()
2306 bw_table = &xhci->rh_bw[i].bw_table; in xhci_setup_port_arrays()
2310 base = &xhci->cap_regs->hc_capbase; in xhci_setup_port_arrays()
2314 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); in xhci_setup_port_arrays()
2326 xhci->ext_caps = kcalloc_node(cap_count, sizeof(*xhci->ext_caps), in xhci_setup_port_arrays()
2328 if (!xhci->ext_caps) in xhci_setup_port_arrays()
2331 xhci->port_caps = kcalloc_node(cap_count, sizeof(*xhci->port_caps), in xhci_setup_port_arrays()
2333 if (!xhci->port_caps) in xhci_setup_port_arrays()
2339 xhci_add_in_port(xhci, num_ports, base + offset, cap_count); in xhci_setup_port_arrays()
2340 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == in xhci_setup_port_arrays()
2346 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { in xhci_setup_port_arrays()
2347 xhci_warn(xhci, "No ports on the roothubs?\n"); in xhci_setup_port_arrays()
2350 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2352 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); in xhci_setup_port_arrays()
2357 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { in xhci_setup_port_arrays()
2358 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2361 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; in xhci_setup_port_arrays()
2363 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { in xhci_setup_port_arrays()
2364 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2367 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; in xhci_setup_port_arrays()
2375 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); in xhci_setup_port_arrays()
2376 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); in xhci_setup_port_arrays()
2381 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) in xhci_mem_init() argument
2384 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_init()
2390 INIT_LIST_HEAD(&xhci->cmd_list); in xhci_mem_init()
2393 INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout); in xhci_mem_init()
2394 init_completion(&xhci->cmd_ring_stop_completion); in xhci_mem_init()
2396 page_size = readl(&xhci->op_regs->page_size); in xhci_mem_init()
2397 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2405 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2408 xhci_warn(xhci, "WARN: no supported page size\n"); in xhci_mem_init()
2410 xhci->page_shift = 12; in xhci_mem_init()
2411 xhci->page_size = 1 << xhci->page_shift; in xhci_mem_init()
2412 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2413 "HCD page size set to %iK", xhci->page_size / 1024); in xhci_mem_init()
2419 val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1)); in xhci_mem_init()
2420 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2422 val2 = readl(&xhci->op_regs->config_reg); in xhci_mem_init()
2424 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2426 writel(val, &xhci->op_regs->config_reg); in xhci_mem_init()
2432 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, in xhci_mem_init()
2434 if (!xhci->dcbaa) in xhci_mem_init()
2436 xhci->dcbaa->dma = dma; in xhci_mem_init()
2437 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2439 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa); in xhci_mem_init()
2440 xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr); in xhci_mem_init()
2449 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, in xhci_mem_init()
2450 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); in xhci_mem_init()
2453 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, in xhci_mem_init()
2454 2112, 64, xhci->page_size); in xhci_mem_init()
2455 if (!xhci->segment_pool || !xhci->device_pool) in xhci_mem_init()
2461 xhci->small_streams_pool = in xhci_mem_init()
2464 xhci->medium_streams_pool = in xhci_mem_init()
2471 if (!xhci->small_streams_pool || !xhci->medium_streams_pool) in xhci_mem_init()
2475 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags); in xhci_mem_init()
2476 if (!xhci->cmd_ring) in xhci_mem_init()
2478 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2479 "Allocated command ring at %p", xhci->cmd_ring); in xhci_mem_init()
2480 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx", in xhci_mem_init()
2481 (unsigned long long)xhci->cmd_ring->first_seg->dma); in xhci_mem_init()
2484 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2486 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | in xhci_mem_init()
2487 xhci->cmd_ring->cycle_state; in xhci_mem_init()
2488 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2490 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2492 xhci->lpm_command = xhci_alloc_command_with_ctx(xhci, true, flags); in xhci_mem_init()
2493 if (!xhci->lpm_command) in xhci_mem_init()
2500 xhci->cmd_ring_reserved_trbs++; in xhci_mem_init()
2502 val = readl(&xhci->cap_regs->db_off); in xhci_mem_init()
2504 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2507 xhci->dba = (void __iomem *) xhci->cap_regs + val; in xhci_mem_init()
2509 xhci->ir_set = &xhci->run_regs->ir_set[0]; in xhci_mem_init()
2515 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); in xhci_mem_init()
2516 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, in xhci_mem_init()
2518 if (!xhci->event_ring) in xhci_mem_init()
2520 if (xhci_check_trb_in_td_math(xhci) < 0) in xhci_mem_init()
2523 ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); in xhci_mem_init()
2528 val = readl(&xhci->ir_set->erst_size); in xhci_mem_init()
2531 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2534 writel(val, &xhci->ir_set->erst_size); in xhci_mem_init()
2536 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2539 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2541 (unsigned long long)xhci->erst.erst_dma_addr); in xhci_mem_init()
2542 val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); in xhci_mem_init()
2544 val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); in xhci_mem_init()
2545 xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); in xhci_mem_init()
2548 xhci_set_hc_event_deq(xhci); in xhci_mem_init()
2549 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2558 xhci->devs[i] = NULL; in xhci_mem_init()
2560 xhci->usb2_rhub.bus_state.resume_done[i] = 0; in xhci_mem_init()
2561 xhci->usb3_rhub.bus_state.resume_done[i] = 0; in xhci_mem_init()
2563 init_completion(&xhci->usb2_rhub.bus_state.rexit_done[i]); in xhci_mem_init()
2564 init_completion(&xhci->usb3_rhub.bus_state.u3exit_done[i]); in xhci_mem_init()
2567 if (scratchpad_alloc(xhci, flags)) in xhci_mem_init()
2569 if (xhci_setup_port_arrays(xhci, flags)) in xhci_mem_init()
2576 temp = readl(&xhci->op_regs->dev_notification); in xhci_mem_init()
2579 writel(temp, &xhci->op_regs->dev_notification); in xhci_mem_init()
2584 xhci_halt(xhci); in xhci_mem_init()
2585 xhci_reset(xhci); in xhci_mem_init()
2586 xhci_mem_cleanup(xhci); in xhci_mem_init()