Lines Matching refs:xhci
28 static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci, in xhci_segment_alloc() argument
36 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_segment_alloc()
42 seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma); in xhci_segment_alloc()
52 dma_pool_free(xhci->segment_pool, seg->trbs, dma); in xhci_segment_alloc()
68 static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg) in xhci_segment_free() argument
71 dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma); in xhci_segment_free()
78 static void xhci_free_segments_for_ring(struct xhci_hcd *xhci, in xhci_free_segments_for_ring() argument
86 xhci_segment_free(xhci, seg); in xhci_free_segments_for_ring()
89 xhci_segment_free(xhci, first); in xhci_free_segments_for_ring()
99 static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev, in xhci_link_segments() argument
117 if (xhci_link_trb_quirk(xhci) || in xhci_link_segments()
119 (xhci->quirks & XHCI_AMD_0x96_HOST))) in xhci_link_segments()
129 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_link_rings() argument
139 xhci_link_segments(xhci, ring->enq_seg, first, ring->type); in xhci_link_rings()
140 xhci_link_segments(xhci, last, next, ring->type); in xhci_link_rings()
277 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring) in xhci_ring_free() argument
287 xhci_free_segments_for_ring(xhci, ring->first_seg); in xhci_ring_free()
318 static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci, in xhci_alloc_segments_for_ring() argument
325 prev = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
334 next = xhci_segment_alloc(xhci, cycle_state, max_packet, flags); in xhci_alloc_segments_for_ring()
339 xhci_segment_free(xhci, prev); in xhci_alloc_segments_for_ring()
344 xhci_link_segments(xhci, prev, next, type); in xhci_alloc_segments_for_ring()
349 xhci_link_segments(xhci, prev, *first, type); in xhci_alloc_segments_for_ring()
362 struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci, in xhci_ring_alloc() argument
368 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_ring_alloc()
381 ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg, in xhci_ring_alloc()
402 void xhci_free_endpoint_ring(struct xhci_hcd *xhci, in xhci_free_endpoint_ring() argument
406 xhci_ring_free(xhci, virt_dev->eps[ep_index].ring); in xhci_free_endpoint_ring()
414 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring, in xhci_ring_expansion() argument
430 ret = xhci_alloc_segments_for_ring(xhci, &first, &last, in xhci_ring_expansion()
443 xhci_segment_free(xhci, first); in xhci_ring_expansion()
451 xhci_link_rings(xhci, ring, first, last, num_segs); in xhci_ring_expansion()
453 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, in xhci_ring_expansion()
460 struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci, in xhci_alloc_container_ctx() argument
464 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_container_ctx()
474 ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024; in xhci_alloc_container_ctx()
476 ctx->size += CTX_SIZE(xhci->hcc_params); in xhci_alloc_container_ctx()
478 ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma); in xhci_alloc_container_ctx()
486 void xhci_free_container_ctx(struct xhci_hcd *xhci, in xhci_free_container_ctx() argument
491 dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma); in xhci_free_container_ctx()
504 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci, in xhci_get_slot_ctx() argument
511 (ctx->bytes + CTX_SIZE(xhci->hcc_params)); in xhci_get_slot_ctx()
514 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci, in xhci_get_ep_ctx() argument
524 (ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params))); in xhci_get_ep_ctx()
530 static void xhci_free_stream_ctx(struct xhci_hcd *xhci, in xhci_free_stream_ctx() argument
534 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_stream_ctx()
541 return dma_pool_free(xhci->small_streams_pool, in xhci_free_stream_ctx()
544 return dma_pool_free(xhci->medium_streams_pool, in xhci_free_stream_ctx()
558 static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci, in xhci_alloc_stream_ctx() argument
562 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_ctx()
569 return dma_pool_alloc(xhci->small_streams_pool, in xhci_alloc_stream_ctx()
572 return dma_pool_alloc(xhci->medium_streams_pool, in xhci_alloc_stream_ctx()
612 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci, in xhci_alloc_stream_info() argument
622 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_stream_info()
624 xhci_dbg(xhci, "Allocating %u streams and %u " in xhci_alloc_stream_info()
627 if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) { in xhci_alloc_stream_info()
628 xhci_dbg(xhci, "Command ring has no reserved TRBs available\n"); in xhci_alloc_stream_info()
631 xhci->cmd_ring_reserved_trbs++; in xhci_alloc_stream_info()
649 stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci, in xhci_alloc_stream_info()
659 xhci_alloc_command_with_ctx(xhci, true, mem_flags); in xhci_alloc_stream_info()
672 xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet, in xhci_alloc_stream_info()
685 xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n", in xhci_alloc_stream_info()
690 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
708 xhci_ring_free(xhci, cur_ring); in xhci_alloc_stream_info()
712 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_alloc_stream_info()
718 xhci->cmd_ring_reserved_trbs--; in xhci_alloc_stream_info()
725 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci, in xhci_setup_streams_ep_input_ctx() argument
735 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change, in xhci_setup_streams_ep_input_ctx()
762 void xhci_free_stream_info(struct xhci_hcd *xhci, in xhci_free_stream_info() argument
775 xhci_ring_free(xhci, cur_ring); in xhci_free_stream_info()
779 xhci_free_command(xhci, stream_info->free_streams_command); in xhci_free_stream_info()
780 xhci->cmd_ring_reserved_trbs--; in xhci_free_stream_info()
782 xhci_free_stream_ctx(xhci, in xhci_free_stream_info()
794 static void xhci_init_endpoint_timer(struct xhci_hcd *xhci, in xhci_init_endpoint_timer() argument
799 ep->xhci = xhci; in xhci_init_endpoint_timer()
802 static void xhci_free_tt_info(struct xhci_hcd *xhci, in xhci_free_tt_info() argument
814 virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_tt_info()
815 xhci_dbg(xhci, "Bad real port.\n"); in xhci_free_tt_info()
819 tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_free_tt_info()
832 int xhci_alloc_tt_info(struct xhci_hcd *xhci, in xhci_alloc_tt_info() argument
840 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_tt_info()
856 &xhci->rh_bw[virt_dev->real_port - 1].tts); in xhci_alloc_tt_info()
867 xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id); in xhci_alloc_tt_info()
877 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_device() argument
884 if (slot_id == 0 || !xhci->devs[slot_id]) in xhci_free_virt_device()
887 dev = xhci->devs[slot_id]; in xhci_free_virt_device()
889 xhci->dcbaa->dev_context_ptrs[slot_id] = 0; in xhci_free_virt_device()
900 xhci_ring_free(xhci, dev->eps[i].ring); in xhci_free_virt_device()
902 xhci_free_stream_info(xhci, in xhci_free_virt_device()
910 xhci_warn(xhci, "Slot %u endpoint %u " in xhci_free_virt_device()
915 xhci_free_tt_info(xhci, dev, slot_id); in xhci_free_virt_device()
917 xhci_update_tt_active_eps(xhci, dev, old_active_eps); in xhci_free_virt_device()
920 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_free_virt_device()
922 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_free_virt_device()
926 kfree(xhci->devs[slot_id]); in xhci_free_virt_device()
927 xhci->devs[slot_id] = NULL; in xhci_free_virt_device()
936 void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id) in xhci_free_virt_devices_depth_first() argument
943 vdev = xhci->devs[slot_id]; in xhci_free_virt_devices_depth_first()
948 vdev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) { in xhci_free_virt_devices_depth_first()
949 xhci_dbg(xhci, "Bad vdev->real_port.\n"); in xhci_free_virt_devices_depth_first()
953 tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts); in xhci_free_virt_devices_depth_first()
958 for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) { in xhci_free_virt_devices_depth_first()
959 vdev = xhci->devs[i]; in xhci_free_virt_devices_depth_first()
962 xhci, i); in xhci_free_virt_devices_depth_first()
968 xhci_debugfs_remove_slot(xhci, slot_id); in xhci_free_virt_devices_depth_first()
969 xhci_free_virt_device(xhci, slot_id); in xhci_free_virt_devices_depth_first()
972 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id, in xhci_alloc_virt_device() argument
979 if (slot_id == 0 || xhci->devs[slot_id]) { in xhci_alloc_virt_device()
980 xhci_warn(xhci, "Bad Slot ID %d\n", slot_id); in xhci_alloc_virt_device()
989 dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags); in xhci_alloc_virt_device()
993 xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
997 dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags); in xhci_alloc_virt_device()
1001 xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id, in xhci_alloc_virt_device()
1006 xhci_init_endpoint_timer(xhci, &dev->eps[i]); in xhci_alloc_virt_device()
1012 dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags); in xhci_alloc_virt_device()
1019 xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma); in xhci_alloc_virt_device()
1020 xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n", in xhci_alloc_virt_device()
1022 &xhci->dcbaa->dev_context_ptrs[slot_id], in xhci_alloc_virt_device()
1023 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id])); in xhci_alloc_virt_device()
1027 xhci->devs[slot_id] = dev; in xhci_alloc_virt_device()
1033 xhci_free_container_ctx(xhci, dev->in_ctx); in xhci_alloc_virt_device()
1035 xhci_free_container_ctx(xhci, dev->out_ctx); in xhci_alloc_virt_device()
1041 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci, in xhci_copy_ep0_dequeue_into_input_ctx() argument
1048 virt_dev = xhci->devs[udev->slot_id]; in xhci_copy_ep0_dequeue_into_input_ctx()
1049 ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0); in xhci_copy_ep0_dequeue_into_input_ctx()
1073 static u32 xhci_find_real_port_number(struct xhci_hcd *xhci, in xhci_find_real_port_number() argument
1080 hcd = xhci->shared_hcd; in xhci_find_real_port_number()
1082 hcd = xhci->main_hcd; in xhci_find_real_port_number()
1092 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev) in xhci_setup_addressable_virt_dev() argument
1101 dev = xhci->devs[udev->slot_id]; in xhci_setup_addressable_virt_dev()
1104 xhci_warn(xhci, "Slot ID %d is not assigned to this device\n", in xhci_setup_addressable_virt_dev()
1108 ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0); in xhci_setup_addressable_virt_dev()
1109 slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx); in xhci_setup_addressable_virt_dev()
1136 xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n"); in xhci_setup_addressable_virt_dev()
1144 port_num = xhci_find_real_port_number(xhci, udev); in xhci_setup_addressable_virt_dev()
1154 xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num); in xhci_setup_addressable_virt_dev()
1155 xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port); in xhci_setup_addressable_virt_dev()
1164 dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table; in xhci_setup_addressable_virt_dev()
1169 rh_bw = &xhci->rh_bw[port_num - 1]; in xhci_setup_addressable_virt_dev()
1184 xhci_warn(xhci, "WARN: Didn't find a matching TT\n"); in xhci_setup_addressable_virt_dev()
1194 xhci_dbg(xhci, "udev->tt = %p\n", udev->tt); in xhci_setup_addressable_virt_dev()
1195 xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport); in xhci_setup_addressable_virt_dev()
1421 int xhci_endpoint_init(struct xhci_hcd *xhci, in xhci_endpoint_init() argument
1441 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_init()
1461 if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) && in xhci_endpoint_init()
1482 if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100) in xhci_endpoint_init()
1485 if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2)) in xhci_endpoint_init()
1490 xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags); in xhci_endpoint_init()
1514 void xhci_endpoint_zero(struct xhci_hcd *xhci, in xhci_endpoint_zero() argument
1522 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index); in xhci_endpoint_zero()
1543 void xhci_update_bw_info(struct xhci_hcd *xhci, in xhci_update_bw_info() argument
1568 ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i); in xhci_update_bw_info()
1601 void xhci_endpoint_copy(struct xhci_hcd *xhci, in xhci_endpoint_copy() argument
1609 out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index); in xhci_endpoint_copy()
1610 in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index); in xhci_endpoint_copy()
1616 if (xhci->quirks & XHCI_MTK_HOST) { in xhci_endpoint_copy()
1627 void xhci_slot_copy(struct xhci_hcd *xhci, in xhci_slot_copy() argument
1634 in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx); in xhci_slot_copy()
1635 out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx); in xhci_slot_copy()
1644 static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags) in scratchpad_alloc() argument
1647 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_alloc()
1648 int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_alloc()
1650 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in scratchpad_alloc()
1656 xhci->scratchpad = kzalloc_node(sizeof(*xhci->scratchpad), flags, in scratchpad_alloc()
1658 if (!xhci->scratchpad) in scratchpad_alloc()
1661 xhci->scratchpad->sp_array = dma_alloc_coherent(dev, in scratchpad_alloc()
1663 &xhci->scratchpad->sp_dma, flags); in scratchpad_alloc()
1664 if (!xhci->scratchpad->sp_array) in scratchpad_alloc()
1667 xhci->scratchpad->sp_buffers = kcalloc_node(num_sp, sizeof(void *), in scratchpad_alloc()
1669 if (!xhci->scratchpad->sp_buffers) in scratchpad_alloc()
1672 xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma); in scratchpad_alloc()
1675 void *buf = dma_zalloc_coherent(dev, xhci->page_size, &dma, in scratchpad_alloc()
1680 xhci->scratchpad->sp_array[i] = dma; in scratchpad_alloc()
1681 xhci->scratchpad->sp_buffers[i] = buf; in scratchpad_alloc()
1688 dma_free_coherent(dev, xhci->page_size, in scratchpad_alloc()
1689 xhci->scratchpad->sp_buffers[i], in scratchpad_alloc()
1690 xhci->scratchpad->sp_array[i]); in scratchpad_alloc()
1693 kfree(xhci->scratchpad->sp_buffers); in scratchpad_alloc()
1697 xhci->scratchpad->sp_array, in scratchpad_alloc()
1698 xhci->scratchpad->sp_dma); in scratchpad_alloc()
1701 kfree(xhci->scratchpad); in scratchpad_alloc()
1702 xhci->scratchpad = NULL; in scratchpad_alloc()
1708 static void scratchpad_free(struct xhci_hcd *xhci) in scratchpad_free() argument
1712 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in scratchpad_free()
1714 if (!xhci->scratchpad) in scratchpad_free()
1717 num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2); in scratchpad_free()
1720 dma_free_coherent(dev, xhci->page_size, in scratchpad_free()
1721 xhci->scratchpad->sp_buffers[i], in scratchpad_free()
1722 xhci->scratchpad->sp_array[i]); in scratchpad_free()
1724 kfree(xhci->scratchpad->sp_buffers); in scratchpad_free()
1726 xhci->scratchpad->sp_array, in scratchpad_free()
1727 xhci->scratchpad->sp_dma); in scratchpad_free()
1728 kfree(xhci->scratchpad); in scratchpad_free()
1729 xhci->scratchpad = NULL; in scratchpad_free()
1732 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci, in xhci_alloc_command() argument
1736 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_alloc_command()
1758 struct xhci_command *xhci_alloc_command_with_ctx(struct xhci_hcd *xhci, in xhci_alloc_command_with_ctx() argument
1763 command = xhci_alloc_command(xhci, allocate_completion, mem_flags); in xhci_alloc_command_with_ctx()
1767 command->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, in xhci_alloc_command_with_ctx()
1782 void xhci_free_command(struct xhci_hcd *xhci, in xhci_free_command() argument
1785 xhci_free_container_ctx(xhci, in xhci_free_command()
1791 int xhci_alloc_erst(struct xhci_hcd *xhci, in xhci_alloc_erst() argument
1802 erst->entries = dma_zalloc_coherent(xhci_to_hcd(xhci)->self.sysdev, in xhci_alloc_erst()
1821 void xhci_free_erst(struct xhci_hcd *xhci, struct xhci_erst *erst) in xhci_free_erst() argument
1824 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_free_erst()
1834 void xhci_mem_cleanup(struct xhci_hcd *xhci) in xhci_mem_cleanup() argument
1836 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_cleanup()
1839 cancel_delayed_work_sync(&xhci->cmd_timer); in xhci_mem_cleanup()
1841 xhci_free_erst(xhci, &xhci->erst); in xhci_mem_cleanup()
1843 if (xhci->event_ring) in xhci_mem_cleanup()
1844 xhci_ring_free(xhci, xhci->event_ring); in xhci_mem_cleanup()
1845 xhci->event_ring = NULL; in xhci_mem_cleanup()
1846 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring"); in xhci_mem_cleanup()
1848 if (xhci->lpm_command) in xhci_mem_cleanup()
1849 xhci_free_command(xhci, xhci->lpm_command); in xhci_mem_cleanup()
1850 xhci->lpm_command = NULL; in xhci_mem_cleanup()
1851 if (xhci->cmd_ring) in xhci_mem_cleanup()
1852 xhci_ring_free(xhci, xhci->cmd_ring); in xhci_mem_cleanup()
1853 xhci->cmd_ring = NULL; in xhci_mem_cleanup()
1854 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring"); in xhci_mem_cleanup()
1855 xhci_cleanup_command_queue(xhci); in xhci_mem_cleanup()
1857 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_mem_cleanup()
1858 for (i = 0; i < num_ports && xhci->rh_bw; i++) { in xhci_mem_cleanup()
1859 struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table; in xhci_mem_cleanup()
1867 for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--) in xhci_mem_cleanup()
1868 xhci_free_virt_devices_depth_first(xhci, i); in xhci_mem_cleanup()
1870 dma_pool_destroy(xhci->segment_pool); in xhci_mem_cleanup()
1871 xhci->segment_pool = NULL; in xhci_mem_cleanup()
1872 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool"); in xhci_mem_cleanup()
1874 dma_pool_destroy(xhci->device_pool); in xhci_mem_cleanup()
1875 xhci->device_pool = NULL; in xhci_mem_cleanup()
1876 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool"); in xhci_mem_cleanup()
1878 dma_pool_destroy(xhci->small_streams_pool); in xhci_mem_cleanup()
1879 xhci->small_streams_pool = NULL; in xhci_mem_cleanup()
1880 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1883 dma_pool_destroy(xhci->medium_streams_pool); in xhci_mem_cleanup()
1884 xhci->medium_streams_pool = NULL; in xhci_mem_cleanup()
1885 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_cleanup()
1888 if (xhci->dcbaa) in xhci_mem_cleanup()
1889 dma_free_coherent(dev, sizeof(*xhci->dcbaa), in xhci_mem_cleanup()
1890 xhci->dcbaa, xhci->dcbaa->dma); in xhci_mem_cleanup()
1891 xhci->dcbaa = NULL; in xhci_mem_cleanup()
1893 scratchpad_free(xhci); in xhci_mem_cleanup()
1895 if (!xhci->rh_bw) in xhci_mem_cleanup()
1900 list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) { in xhci_mem_cleanup()
1907 xhci->cmd_ring_reserved_trbs = 0; in xhci_mem_cleanup()
1908 xhci->usb2_rhub.num_ports = 0; in xhci_mem_cleanup()
1909 xhci->usb3_rhub.num_ports = 0; in xhci_mem_cleanup()
1910 xhci->num_active_eps = 0; in xhci_mem_cleanup()
1911 kfree(xhci->usb2_rhub.ports); in xhci_mem_cleanup()
1912 kfree(xhci->usb3_rhub.ports); in xhci_mem_cleanup()
1913 kfree(xhci->hw_ports); in xhci_mem_cleanup()
1914 kfree(xhci->rh_bw); in xhci_mem_cleanup()
1915 kfree(xhci->ext_caps); in xhci_mem_cleanup()
1917 xhci->usb2_rhub.ports = NULL; in xhci_mem_cleanup()
1918 xhci->usb3_rhub.ports = NULL; in xhci_mem_cleanup()
1919 xhci->hw_ports = NULL; in xhci_mem_cleanup()
1920 xhci->rh_bw = NULL; in xhci_mem_cleanup()
1921 xhci->ext_caps = NULL; in xhci_mem_cleanup()
1923 xhci->page_size = 0; in xhci_mem_cleanup()
1924 xhci->page_shift = 0; in xhci_mem_cleanup()
1925 xhci->bus_state[0].bus_suspended = 0; in xhci_mem_cleanup()
1926 xhci->bus_state[1].bus_suspended = 0; in xhci_mem_cleanup()
1929 static int xhci_test_trb_in_td(struct xhci_hcd *xhci, in xhci_test_trb_in_td() argument
1944 seg = trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, false); in xhci_test_trb_in_td()
1946 xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n", in xhci_test_trb_in_td()
1948 xhci_warn(xhci, "Tested TRB math w/ seg %p and " in xhci_test_trb_in_td()
1952 xhci_warn(xhci, "starting TRB %p (0x%llx DMA), " in xhci_test_trb_in_td()
1956 xhci_warn(xhci, "Expected seg %p, got seg %p\n", in xhci_test_trb_in_td()
1958 trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, in xhci_test_trb_in_td()
1966 static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci) in xhci_check_trb_in_td_math() argument
1975 { xhci->event_ring->first_seg->dma - 16, NULL }, in xhci_check_trb_in_td_math()
1977 { xhci->event_ring->first_seg->dma - 1, NULL }, in xhci_check_trb_in_td_math()
1979 { xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
1981 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16, in xhci_check_trb_in_td_math()
1982 xhci->event_ring->first_seg }, in xhci_check_trb_in_td_math()
1984 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL }, in xhci_check_trb_in_td_math()
1986 { xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL }, in xhci_check_trb_in_td_math()
1998 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
1999 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2000 .end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2001 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2005 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2006 .start_trb = xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2007 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2008 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2012 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2013 .start_trb = xhci->cmd_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2014 .end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2015 .input_dma = xhci->cmd_ring->first_seg->dma, in xhci_check_trb_in_td_math()
2019 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2020 .start_trb = &xhci->event_ring->first_seg->trbs[0], in xhci_check_trb_in_td_math()
2021 .end_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2022 .input_dma = xhci->event_ring->first_seg->dma + 4*16, in xhci_check_trb_in_td_math()
2026 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2027 .start_trb = &xhci->event_ring->first_seg->trbs[3], in xhci_check_trb_in_td_math()
2028 .end_trb = &xhci->event_ring->first_seg->trbs[6], in xhci_check_trb_in_td_math()
2029 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2033 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2034 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2035 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2036 .input_dma = xhci->event_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2040 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2041 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2042 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2043 .input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16, in xhci_check_trb_in_td_math()
2047 { .input_seg = xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2048 .start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3], in xhci_check_trb_in_td_math()
2049 .end_trb = &xhci->event_ring->first_seg->trbs[1], in xhci_check_trb_in_td_math()
2050 .input_dma = xhci->cmd_ring->first_seg->dma + 2*16, in xhci_check_trb_in_td_math()
2060 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2061 xhci->event_ring->first_seg, in xhci_check_trb_in_td_math()
2062 xhci->event_ring->first_seg->trbs, in xhci_check_trb_in_td_math()
2063 &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1], in xhci_check_trb_in_td_math()
2073 ret = xhci_test_trb_in_td(xhci, in xhci_check_trb_in_td_math()
2083 xhci_dbg(xhci, "TRB math tests passed.\n"); in xhci_check_trb_in_td_math()
2087 static void xhci_set_hc_event_deq(struct xhci_hcd *xhci) in xhci_set_hc_event_deq() argument
2092 deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg, in xhci_set_hc_event_deq()
2093 xhci->event_ring->dequeue); in xhci_set_hc_event_deq()
2095 xhci_warn(xhci, "WARN something wrong with SW event ring " in xhci_set_hc_event_deq()
2098 temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2104 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_set_hc_event_deq()
2107 xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp, in xhci_set_hc_event_deq()
2108 &xhci->ir_set->erst_dequeue); in xhci_set_hc_event_deq()
2111 static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports, in xhci_add_in_port() argument
2118 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_add_in_port()
2125 rhub = &xhci->usb3_rhub; in xhci_add_in_port()
2127 rhub = &xhci->usb2_rhub; in xhci_add_in_port()
2129 xhci_warn(xhci, "Ignoring unknown port speed, " in xhci_add_in_port()
2144 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2171 xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n", in xhci_add_in_port()
2181 if (major_revision < 0x03 && xhci->num_ext_caps < max_caps) in xhci_add_in_port()
2182 xhci->ext_caps[xhci->num_ext_caps++] = temp; in xhci_add_in_port()
2185 if ((xhci->hci_version == 0x96) && (major_revision != 0x03) && in xhci_add_in_port()
2187 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2189 xhci->sw_lpm_support = 1; in xhci_add_in_port()
2192 if ((xhci->hci_version >= 0x100) && (major_revision != 0x03)) { in xhci_add_in_port()
2193 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2195 xhci->sw_lpm_support = 1; in xhci_add_in_port()
2197 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_add_in_port()
2199 xhci->hw_lpm_support = 1; in xhci_add_in_port()
2205 struct xhci_port *hw_port = &xhci->hw_ports[i]; in xhci_add_in_port()
2208 xhci_warn(xhci, "Duplicate port entry, Ext Cap %p," in xhci_add_in_port()
2210 xhci_warn(xhci, "Port was marked as USB %u, " in xhci_add_in_port()
2229 static void xhci_create_rhub_port_array(struct xhci_hcd *xhci, in xhci_create_rhub_port_array() argument
2234 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_create_rhub_port_array()
2240 for (i = 0; i < HCS_MAX_PORTS(xhci->hcs_params1); i++) { in xhci_create_rhub_port_array()
2241 if (xhci->hw_ports[i].rhub != rhub || in xhci_create_rhub_port_array()
2242 xhci->hw_ports[i].hcd_portnum == DUPLICATE_ENTRY) in xhci_create_rhub_port_array()
2244 xhci->hw_ports[i].hcd_portnum = port_index; in xhci_create_rhub_port_array()
2245 rhub->ports[port_index] = &xhci->hw_ports[i]; in xhci_create_rhub_port_array()
2259 static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags) in xhci_setup_port_arrays() argument
2267 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_setup_port_arrays()
2269 num_ports = HCS_MAX_PORTS(xhci->hcs_params1); in xhci_setup_port_arrays()
2270 xhci->hw_ports = kcalloc_node(num_ports, sizeof(*xhci->hw_ports), in xhci_setup_port_arrays()
2272 if (!xhci->hw_ports) in xhci_setup_port_arrays()
2276 xhci->hw_ports[i].addr = &xhci->op_regs->port_status_base + in xhci_setup_port_arrays()
2278 xhci->hw_ports[i].hw_portnum = i; in xhci_setup_port_arrays()
2281 xhci->rh_bw = kcalloc_node(num_ports, sizeof(*xhci->rh_bw), flags, in xhci_setup_port_arrays()
2283 if (!xhci->rh_bw) in xhci_setup_port_arrays()
2288 INIT_LIST_HEAD(&xhci->rh_bw[i].tts); in xhci_setup_port_arrays()
2289 bw_table = &xhci->rh_bw[i].bw_table; in xhci_setup_port_arrays()
2293 base = &xhci->cap_regs->hc_capbase; in xhci_setup_port_arrays()
2297 xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n"); in xhci_setup_port_arrays()
2309 xhci->ext_caps = kcalloc_node(cap_count, sizeof(*xhci->ext_caps), in xhci_setup_port_arrays()
2311 if (!xhci->ext_caps) in xhci_setup_port_arrays()
2317 xhci_add_in_port(xhci, num_ports, base + offset, cap_count); in xhci_setup_port_arrays()
2318 if (xhci->usb2_rhub.num_ports + xhci->usb3_rhub.num_ports == in xhci_setup_port_arrays()
2324 if (xhci->usb2_rhub.num_ports == 0 && xhci->usb3_rhub.num_ports == 0) { in xhci_setup_port_arrays()
2325 xhci_warn(xhci, "No ports on the roothubs?\n"); in xhci_setup_port_arrays()
2328 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2330 xhci->usb2_rhub.num_ports, xhci->usb3_rhub.num_ports); in xhci_setup_port_arrays()
2335 if (xhci->usb3_rhub.num_ports > USB_SS_MAXPORTS) { in xhci_setup_port_arrays()
2336 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2339 xhci->usb3_rhub.num_ports = USB_SS_MAXPORTS; in xhci_setup_port_arrays()
2341 if (xhci->usb2_rhub.num_ports > USB_MAXCHILDREN) { in xhci_setup_port_arrays()
2342 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_setup_port_arrays()
2345 xhci->usb2_rhub.num_ports = USB_MAXCHILDREN; in xhci_setup_port_arrays()
2353 xhci_create_rhub_port_array(xhci, &xhci->usb2_rhub, flags); in xhci_setup_port_arrays()
2354 xhci_create_rhub_port_array(xhci, &xhci->usb3_rhub, flags); in xhci_setup_port_arrays()
2359 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags) in xhci_mem_init() argument
2362 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; in xhci_mem_init()
2368 INIT_LIST_HEAD(&xhci->cmd_list); in xhci_mem_init()
2371 INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout); in xhci_mem_init()
2372 init_completion(&xhci->cmd_ring_stop_completion); in xhci_mem_init()
2374 page_size = readl(&xhci->op_regs->page_size); in xhci_mem_init()
2375 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2383 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2386 xhci_warn(xhci, "WARN: no supported page size\n"); in xhci_mem_init()
2388 xhci->page_shift = 12; in xhci_mem_init()
2389 xhci->page_size = 1 << xhci->page_shift; in xhci_mem_init()
2390 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2391 "HCD page size set to %iK", xhci->page_size / 1024); in xhci_mem_init()
2397 val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1)); in xhci_mem_init()
2398 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2400 val2 = readl(&xhci->op_regs->config_reg); in xhci_mem_init()
2402 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2404 writel(val, &xhci->op_regs->config_reg); in xhci_mem_init()
2410 xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma, in xhci_mem_init()
2412 if (!xhci->dcbaa) in xhci_mem_init()
2414 memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa)); in xhci_mem_init()
2415 xhci->dcbaa->dma = dma; in xhci_mem_init()
2416 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2418 (unsigned long long)xhci->dcbaa->dma, xhci->dcbaa); in xhci_mem_init()
2419 xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr); in xhci_mem_init()
2428 xhci->segment_pool = dma_pool_create("xHCI ring segments", dev, in xhci_mem_init()
2429 TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size); in xhci_mem_init()
2432 xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev, in xhci_mem_init()
2433 2112, 64, xhci->page_size); in xhci_mem_init()
2434 if (!xhci->segment_pool || !xhci->device_pool) in xhci_mem_init()
2440 xhci->small_streams_pool = in xhci_mem_init()
2443 xhci->medium_streams_pool = in xhci_mem_init()
2450 if (!xhci->small_streams_pool || !xhci->medium_streams_pool) in xhci_mem_init()
2454 xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags); in xhci_mem_init()
2455 if (!xhci->cmd_ring) in xhci_mem_init()
2457 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2458 "Allocated command ring at %p", xhci->cmd_ring); in xhci_mem_init()
2459 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx", in xhci_mem_init()
2460 (unsigned long long)xhci->cmd_ring->first_seg->dma); in xhci_mem_init()
2463 val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2465 (xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) | in xhci_mem_init()
2466 xhci->cmd_ring->cycle_state; in xhci_mem_init()
2467 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2469 xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring); in xhci_mem_init()
2471 xhci->lpm_command = xhci_alloc_command_with_ctx(xhci, true, flags); in xhci_mem_init()
2472 if (!xhci->lpm_command) in xhci_mem_init()
2479 xhci->cmd_ring_reserved_trbs++; in xhci_mem_init()
2481 val = readl(&xhci->cap_regs->db_off); in xhci_mem_init()
2483 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2486 xhci->dba = (void __iomem *) xhci->cap_regs + val; in xhci_mem_init()
2488 xhci->ir_set = &xhci->run_regs->ir_set[0]; in xhci_mem_init()
2494 xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring"); in xhci_mem_init()
2495 xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT, in xhci_mem_init()
2497 if (!xhci->event_ring) in xhci_mem_init()
2499 if (xhci_check_trb_in_td_math(xhci) < 0) in xhci_mem_init()
2502 ret = xhci_alloc_erst(xhci, xhci->event_ring, &xhci->erst, flags); in xhci_mem_init()
2507 val = readl(&xhci->ir_set->erst_size); in xhci_mem_init()
2510 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2513 writel(val, &xhci->ir_set->erst_size); in xhci_mem_init()
2515 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2518 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2520 (unsigned long long)xhci->erst.erst_dma_addr); in xhci_mem_init()
2521 val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base); in xhci_mem_init()
2523 val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK); in xhci_mem_init()
2524 xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base); in xhci_mem_init()
2527 xhci_set_hc_event_deq(xhci); in xhci_mem_init()
2528 xhci_dbg_trace(xhci, trace_xhci_dbg_init, in xhci_mem_init()
2537 xhci->devs[i] = NULL; in xhci_mem_init()
2539 xhci->bus_state[0].resume_done[i] = 0; in xhci_mem_init()
2540 xhci->bus_state[1].resume_done[i] = 0; in xhci_mem_init()
2542 init_completion(&xhci->bus_state[1].rexit_done[i]); in xhci_mem_init()
2545 if (scratchpad_alloc(xhci, flags)) in xhci_mem_init()
2547 if (xhci_setup_port_arrays(xhci, flags)) in xhci_mem_init()
2554 temp = readl(&xhci->op_regs->dev_notification); in xhci_mem_init()
2557 writel(temp, &xhci->op_regs->dev_notification); in xhci_mem_init()
2562 xhci_halt(xhci); in xhci_mem_init()
2563 xhci_reset(xhci); in xhci_mem_init()
2564 xhci_mem_cleanup(xhci); in xhci_mem_init()