39 static inline struct tb *tcm_to_tb(struct tb_cm *tcm)  in tcm_to_tb()
41 	return ((void *)tcm - sizeof(struct tb));  in tcm_to_tb()
46 	struct tb *tb;  member
54 static void tb_queue_hotplug(struct tb *tb, u64 route, u8 port, bool unplug)  in tb_queue_hotplug()  argument
62 	ev->tb = tb;  in tb_queue_hotplug()
67 	queue_work(tb->wq, &ev->work);  in tb_queue_hotplug()
74 	struct tb_cm *tcm = tb_priv(sw->tb);  in tb_add_dp_resources()
91 	struct tb_cm *tcm = tb_priv(sw->tb);  in tb_remove_dp_resources()
108 static void tb_discover_dp_resource(struct tb *tb, struct tb_port *port)  in tb_discover_dp_resource()  argument
110 	struct tb_cm *tcm = tb_priv(tb);  in tb_discover_dp_resource()
123 static void tb_discover_dp_resources(struct tb *tb)  in tb_discover_dp_resources()  argument
125 	struct tb_cm *tcm = tb_priv(tb);  in tb_discover_dp_resources()
130 			tb_discover_dp_resource(tb, tunnel->dst_port);  in tb_discover_dp_resources()
138 	struct tb *tb = sw->tb;  in tb_switch_discover_tunnels()  local
146 			tunnel = tb_tunnel_discover_dp(tb, port, alloc_hopids);  in tb_switch_discover_tunnels()
152 				tb_switch_enable_tmu_1st_child(tb->root_switch,  in tb_switch_discover_tunnels()
157 			tunnel = tb_tunnel_discover_pci(tb, port, alloc_hopids);  in tb_switch_discover_tunnels()
161 			tunnel = tb_tunnel_discover_usb3(tb, port, alloc_hopids);  in tb_switch_discover_tunnels()
180 static void tb_discover_tunnels(struct tb *tb)  in tb_discover_tunnels()  argument
182 	struct tb_cm *tcm = tb_priv(tb);  in tb_discover_tunnels()
185 	tb_switch_discover_tunnels(tb->root_switch, &tcm->tunnel_list, true);  in tb_discover_tunnels()
223 	struct tb *tb = sw->tb;  in tb_scan_xdomain()  local
231 	xd = tb_xdomain_find_by_route(tb, route);  in tb_scan_xdomain()
237 	xd = tb_xdomain_alloc(tb, &sw->dev, route, tb->root_switch->uuid,  in tb_scan_xdomain()
300 static struct tb_tunnel *tb_find_tunnel(struct tb *tb, enum tb_tunnel_type type,  in tb_find_tunnel()  argument
304 	struct tb_cm *tcm = tb_priv(tb);  in tb_find_tunnel()
318 static struct tb_tunnel *tb_find_first_usb3_tunnel(struct tb *tb,  in tb_find_first_usb3_tunnel()  argument
332 	if (sw == tb->root_switch)  in tb_find_first_usb3_tunnel()
336 	port = tb_port_at(tb_route(sw), tb->root_switch);  in tb_find_first_usb3_tunnel()
338 	usb3_down = usb4_switch_map_usb3_down(tb->root_switch, port);  in tb_find_first_usb3_tunnel()
342 	return tb_find_tunnel(tb, TB_TUNNEL_USB3, usb3_down, NULL);  in tb_find_first_usb3_tunnel()
345 static int tb_available_bandwidth(struct tb *tb, struct tb_port *src_port,  in tb_available_bandwidth()  argument
349 	struct tb_cm *tcm = tb_priv(tb);  in tb_available_bandwidth()
355 	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);  in tb_available_bandwidth()
438 static int tb_release_unused_usb3_bandwidth(struct tb *tb,  in tb_release_unused_usb3_bandwidth()  argument
444 	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);  in tb_release_unused_usb3_bandwidth()
448 static void tb_reclaim_usb3_bandwidth(struct tb *tb, struct tb_port *src_port,  in tb_reclaim_usb3_bandwidth()  argument
454 	tunnel = tb_find_first_usb3_tunnel(tb, src_port, dst_port);  in tb_reclaim_usb3_bandwidth()
458 	tb_dbg(tb, "reclaiming unused bandwidth for USB3\n");  in tb_reclaim_usb3_bandwidth()
464 	ret = tb_available_bandwidth(tb, tunnel->src_port, tunnel->dst_port,  in tb_reclaim_usb3_bandwidth()
467 		tb_warn(tb, "failed to calculate available bandwidth\n");  in tb_reclaim_usb3_bandwidth()
471 	tb_dbg(tb, "available bandwidth for USB3 %d/%d Mb/s\n",  in tb_reclaim_usb3_bandwidth()
477 static int tb_tunnel_usb3(struct tb *tb, struct tb_switch *sw)  in tb_tunnel_usb3()  argument
482 	struct tb_cm *tcm = tb_priv(tb);  in tb_tunnel_usb3()
486 		tb_dbg(tb, "USB3 tunneling disabled, not creating tunnel\n");  in tb_tunnel_usb3()
518 		ret = tb_release_unused_usb3_bandwidth(tb, down, up);  in tb_tunnel_usb3()
523 	ret = tb_available_bandwidth(tb, down, up, &available_up,  in tb_tunnel_usb3()
531 	tunnel = tb_tunnel_alloc_usb3(tb, up, down, available_up,  in tb_tunnel_usb3()
547 		tb_reclaim_usb3_bandwidth(tb, down, up);  in tb_tunnel_usb3()
555 		tb_reclaim_usb3_bandwidth(tb, down, up);  in tb_tunnel_usb3()
569 		ret = tb_tunnel_usb3(sw->tb, sw);  in tb_create_usb3_tunnels()
608 	struct tb_cm *tcm = tb_priv(port->sw->tb);  in tb_scan_port()
619 		tb_queue_hotplug(port->sw->tb, tb_route(port->sw), port->port,  in tb_scan_port()
640 	sw = tb_switch_alloc(port->sw->tb, &port->sw->dev,  in tb_scan_port()
731 	if (tcm->hotplug_active && tb_tunnel_usb3(sw->tb, sw))  in tb_scan_port()
741 	struct tb *tb;  in tb_deactivate_and_free_tunnel()  local
749 	tb = tunnel->tb;  in tb_deactivate_and_free_tunnel()
768 		tb_reclaim_usb3_bandwidth(tb, src_port, dst_port);  in tb_deactivate_and_free_tunnel()
785 static void tb_free_invalid_tunnels(struct tb *tb)  in tb_free_invalid_tunnels()  argument
787 	struct tb_cm *tcm = tb_priv(tb);  in tb_free_invalid_tunnels()
872 static struct tb_port *tb_find_dp_out(struct tb *tb, struct tb_port *in)  in tb_find_dp_out()  argument
875 	struct tb_cm *tcm = tb_priv(tb);  in tb_find_dp_out()
878 		tb_port_at(tb_route(in->sw), tb->root_switch) : NULL;  in tb_find_dp_out()
898 			p = tb_port_at(tb_route(port->sw), tb->root_switch);  in tb_find_dp_out()
909 static void tb_tunnel_dp(struct tb *tb)  in tb_tunnel_dp()  argument
912 	struct tb_cm *tcm = tb_priv(tb);  in tb_tunnel_dp()
917 		tb_dbg(tb, "DP tunneling disabled, not creating tunnel\n");  in tb_tunnel_dp()
925 	tb_dbg(tb, "looking for DP IN <-> DP OUT pairs:\n");  in tb_tunnel_dp()
940 		out = tb_find_dp_out(tb, port);  in tb_tunnel_dp()
948 		tb_dbg(tb, "no suitable DP IN adapter available, not tunneling\n");  in tb_tunnel_dp()
952 		tb_dbg(tb, "no suitable DP OUT adapter available, not tunneling\n");  in tb_tunnel_dp()
987 	ret = tb_release_unused_usb3_bandwidth(tb, in, out);  in tb_tunnel_dp()
989 		tb_warn(tb, "failed to release unused bandwidth\n");  in tb_tunnel_dp()
993 	ret = tb_available_bandwidth(tb, in, out, &available_up,  in tb_tunnel_dp()
998 	tb_dbg(tb, "available bandwidth for new DP tunnel %u/%u Mb/s\n",  in tb_tunnel_dp()
1001 	tunnel = tb_tunnel_alloc_dp(tb, in, out, link_nr, available_up,  in tb_tunnel_dp()
1014 	tb_reclaim_usb3_bandwidth(tb, in, out);  in tb_tunnel_dp()
1019 	tb_switch_enable_tmu_1st_child(tb->root_switch, TB_SWITCH_TMU_RATE_HIFI);  in tb_tunnel_dp()
1026 	tb_reclaim_usb3_bandwidth(tb, in, out);  in tb_tunnel_dp()
1036 static void tb_dp_resource_unavailable(struct tb *tb, struct tb_port *port)  in tb_dp_resource_unavailable()  argument
1051 	tunnel = tb_find_tunnel(tb, TB_TUNNEL_DP, in, out);  in tb_dp_resource_unavailable()
1059 	tb_tunnel_dp(tb);  in tb_dp_resource_unavailable()
1062 static void tb_dp_resource_available(struct tb *tb, struct tb_port *port)  in tb_dp_resource_available()  argument
1064 	struct tb_cm *tcm = tb_priv(tb);  in tb_dp_resource_available()
1080 	tb_tunnel_dp(tb);  in tb_dp_resource_available()
1083 static void tb_disconnect_and_release_dp(struct tb *tb)  in tb_disconnect_and_release_dp()  argument
1085 	struct tb_cm *tcm = tb_priv(tb);  in tb_disconnect_and_release_dp()
1106 static int tb_disconnect_pci(struct tb *tb, struct tb_switch *sw)  in tb_disconnect_pci()  argument
1115 	tunnel = tb_find_tunnel(tb, TB_TUNNEL_PCI, NULL, up);  in tb_disconnect_pci()
1127 static int tb_tunnel_pci(struct tb *tb, struct tb_switch *sw)  in tb_tunnel_pci()  argument
1130 	struct tb_cm *tcm = tb_priv(tb);  in tb_tunnel_pci()
1148 	tunnel = tb_tunnel_alloc_pci(tb, up, down);  in tb_tunnel_pci()
1173 static int tb_approve_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,  in tb_approve_xdomain_paths()  argument
1177 	struct tb_cm *tcm = tb_priv(tb);  in tb_approve_xdomain_paths()
1184 	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);  in tb_approve_xdomain_paths()
1186 	mutex_lock(&tb->lock);  in tb_approve_xdomain_paths()
1187 	tunnel = tb_tunnel_alloc_dma(tb, nhi_port, dst_port, transmit_path,  in tb_approve_xdomain_paths()
1190 		mutex_unlock(&tb->lock);  in tb_approve_xdomain_paths()
1198 		mutex_unlock(&tb->lock);  in tb_approve_xdomain_paths()
1203 	mutex_unlock(&tb->lock);  in tb_approve_xdomain_paths()
1207 static void __tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,  in __tb_disconnect_xdomain_paths()  argument
1211 	struct tb_cm *tcm = tb_priv(tb);  in __tb_disconnect_xdomain_paths()
1218 	nhi_port = tb_switch_find_port(tb->root_switch, TB_TYPE_NHI);  in __tb_disconnect_xdomain_paths()
1232 static int tb_disconnect_xdomain_paths(struct tb *tb, struct tb_xdomain *xd,  in tb_disconnect_xdomain_paths()  argument
1237 		mutex_lock(&tb->lock);  in tb_disconnect_xdomain_paths()
1238 		__tb_disconnect_xdomain_paths(tb, xd, transmit_path,  in tb_disconnect_xdomain_paths()
1241 		mutex_unlock(&tb->lock);  in tb_disconnect_xdomain_paths()
1256 	struct tb *tb = ev->tb;  in tb_handle_hotplug()  local
1257 	struct tb_cm *tcm = tb_priv(tb);  in tb_handle_hotplug()
1262 	pm_runtime_get_sync(&tb->dev);  in tb_handle_hotplug()
1264 	mutex_lock(&tb->lock);  in tb_handle_hotplug()
1268 	sw = tb_switch_find_by_route(tb, ev->route);  in tb_handle_hotplug()
1270 		tb_warn(tb,  in tb_handle_hotplug()
1276 		tb_warn(tb,  in tb_handle_hotplug()
1283 		tb_dbg(tb, "hotplug event for upstream port %llx:%x (unplug: %d)\n",  in tb_handle_hotplug()
1296 			tb_free_invalid_tunnels(tb);  in tb_handle_hotplug()
1306 			tb_tunnel_dp(tb);  in tb_handle_hotplug()
1321 			__tb_disconnect_xdomain_paths(tb, xd, -1, -1, -1, -1);  in tb_handle_hotplug()
1325 			tb_dp_resource_unavailable(tb, port);  in tb_handle_hotplug()
1345 			tb_dp_resource_available(tb, port);  in tb_handle_hotplug()
1355 	mutex_unlock(&tb->lock);  in tb_handle_hotplug()
1357 	pm_runtime_mark_last_busy(&tb->dev);  in tb_handle_hotplug()
1358 	pm_runtime_put_autosuspend(&tb->dev);  in tb_handle_hotplug()
1368 static void tb_handle_event(struct tb *tb, enum tb_cfg_pkg_type type,  in tb_handle_event()  argument
1375 		tb_warn(tb, "unexpected event %#x, ignoring\n", type);  in tb_handle_event()
1381 	if (tb_cfg_ack_plug(tb->ctl, route, pkg->port, pkg->unplug)) {  in tb_handle_event()
1382 		tb_warn(tb, "could not ack plug event on %llx:%x\n", route,  in tb_handle_event()
1386 	tb_queue_hotplug(tb, route, pkg->port, pkg->unplug);  in tb_handle_event()
1389 static void tb_stop(struct tb *tb)  in tb_stop()  argument
1391 	struct tb_cm *tcm = tb_priv(tb);  in tb_stop()
1407 	tb_switch_remove(tb->root_switch);  in tb_stop()
1432 static int tb_start(struct tb *tb)  in tb_start()  argument
1434 	struct tb_cm *tcm = tb_priv(tb);  in tb_start()
1437 	tb->root_switch = tb_switch_alloc(tb, &tb->dev, 0);  in tb_start()
1438 	if (IS_ERR(tb->root_switch))  in tb_start()
1439 		return PTR_ERR(tb->root_switch);  in tb_start()
1449 	tb->root_switch->no_nvm_upgrade = !tb_switch_is_usb4(tb->root_switch);  in tb_start()
1451 	tb->root_switch->rpm = tb_switch_is_usb4(tb->root_switch);  in tb_start()
1453 	ret = tb_switch_configure(tb->root_switch);  in tb_start()
1455 		tb_switch_put(tb->root_switch);  in tb_start()
1460 	ret = tb_switch_add(tb->root_switch);  in tb_start()
1462 		tb_switch_put(tb->root_switch);  in tb_start()
1470 	tb_switch_tmu_configure(tb->root_switch, TB_SWITCH_TMU_RATE_NORMAL,  in tb_start()
1473 	tb_switch_tmu_enable(tb->root_switch);  in tb_start()
1475 	tb_scan_switch(tb->root_switch);  in tb_start()
1477 	tb_discover_tunnels(tb);  in tb_start()
1479 	tb_discover_dp_resources(tb);  in tb_start()
1484 	tb_create_usb3_tunnels(tb->root_switch);  in tb_start()
1486 	tb_add_dp_resources(tb->root_switch);  in tb_start()
1488 	device_for_each_child(&tb->root_switch->dev, NULL,  in tb_start()
1496 static int tb_suspend_noirq(struct tb *tb)  in tb_suspend_noirq()  argument
1498 	struct tb_cm *tcm = tb_priv(tb);  in tb_suspend_noirq()
1500 	tb_dbg(tb, "suspending...\n");  in tb_suspend_noirq()
1501 	tb_disconnect_and_release_dp(tb);  in tb_suspend_noirq()
1502 	tb_switch_suspend(tb->root_switch, false);  in tb_suspend_noirq()
1504 	tb_dbg(tb, "suspend finished\n");  in tb_suspend_noirq()
1555 static int tb_resume_noirq(struct tb *tb)  in tb_resume_noirq()  argument
1557 	struct tb_cm *tcm = tb_priv(tb);  in tb_resume_noirq()
1562 	tb_dbg(tb, "resuming...\n");  in tb_resume_noirq()
1565 	tb_switch_reset(tb->root_switch);  in tb_resume_noirq()
1567 	tb_switch_resume(tb->root_switch);  in tb_resume_noirq()
1568 	tb_free_invalid_tunnels(tb);  in tb_resume_noirq()
1569 	tb_free_unplugged_children(tb->root_switch);  in tb_resume_noirq()
1570 	tb_restore_children(tb->root_switch);  in tb_resume_noirq()
1578 	tb_switch_discover_tunnels(tb->root_switch, &tunnels, false);  in tb_resume_noirq()
1601 		tb_dbg(tb, "tunnels restarted, sleeping for 100ms\n");  in tb_resume_noirq()
1606 	tb_dbg(tb, "resume finished\n");  in tb_resume_noirq()
1633 static int tb_freeze_noirq(struct tb *tb)  in tb_freeze_noirq()  argument
1635 	struct tb_cm *tcm = tb_priv(tb);  in tb_freeze_noirq()
1641 static int tb_thaw_noirq(struct tb *tb)  in tb_thaw_noirq()  argument
1643 	struct tb_cm *tcm = tb_priv(tb);  in tb_thaw_noirq()
1649 static void tb_complete(struct tb *tb)  in tb_complete()  argument
1656 	mutex_lock(&tb->lock);  in tb_complete()
1657 	if (tb_free_unplugged_xdomains(tb->root_switch))  in tb_complete()
1658 		tb_scan_switch(tb->root_switch);  in tb_complete()
1659 	mutex_unlock(&tb->lock);  in tb_complete()
1662 static int tb_runtime_suspend(struct tb *tb)  in tb_runtime_suspend()  argument
1664 	struct tb_cm *tcm = tb_priv(tb);  in tb_runtime_suspend()
1666 	mutex_lock(&tb->lock);  in tb_runtime_suspend()
1667 	tb_switch_suspend(tb->root_switch, true);  in tb_runtime_suspend()
1669 	mutex_unlock(&tb->lock);  in tb_runtime_suspend()
1677 	struct tb *tb = tcm_to_tb(tcm);  in tb_remove_work()  local
1679 	mutex_lock(&tb->lock);  in tb_remove_work()
1680 	if (tb->root_switch) {  in tb_remove_work()
1681 		tb_free_unplugged_children(tb->root_switch);  in tb_remove_work()
1682 		tb_free_unplugged_xdomains(tb->root_switch);  in tb_remove_work()
1684 	mutex_unlock(&tb->lock);  in tb_remove_work()
1687 static int tb_runtime_resume(struct tb *tb)  in tb_runtime_resume()  argument
1689 	struct tb_cm *tcm = tb_priv(tb);  in tb_runtime_resume()
1692 	mutex_lock(&tb->lock);  in tb_runtime_resume()
1693 	tb_switch_resume(tb->root_switch);  in tb_runtime_resume()
1694 	tb_free_invalid_tunnels(tb);  in tb_runtime_resume()
1695 	tb_restore_children(tb->root_switch);  in tb_runtime_resume()
1699 	mutex_unlock(&tb->lock);  in tb_runtime_resume()
1706 	queue_delayed_work(tb->wq, &tcm->remove_work, msecs_to_jiffies(50));  in tb_runtime_resume()
1790 struct tb *tb_probe(struct tb_nhi *nhi)  in tb_probe()
1793 	struct tb *tb;  in tb_probe()  local
1795 	tb = tb_domain_alloc(nhi, TB_TIMEOUT, sizeof(*tcm));  in tb_probe()
1796 	if (!tb)  in tb_probe()
1800 		tb->security_level = TB_SECURITY_USER;  in tb_probe()
1802 		tb->security_level = TB_SECURITY_NOPCIE;  in tb_probe()
1804 	tb->cm_ops = &tb_cm_ops;  in tb_probe()
1806 	tcm = tb_priv(tb);  in tb_probe()
1811 	tb_dbg(tb, "using software connection manager\n");  in tb_probe()
1816 	return tb;  in tb_probe()