3  * Enable PCIe link L0s/L1 state and Clock Power Management
46 	struct pci_dev *pdev;		/* Upstream component of the Link */
48 	struct pcie_link_state *root;	/* pointer to the root port link */
49 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
109 static int policy_to_aspm_state(struct pcie_link_state *link)  in policy_to_aspm_state()  argument
122 		return link->aspm_default;  in policy_to_aspm_state()
127 static int policy_to_clkpm_state(struct pcie_link_state *link)  in policy_to_clkpm_state()  argument
138 		return link->clkpm_default;  in policy_to_clkpm_state()
143 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)  in pcie_set_clkpm_nocheck()  argument
146 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_set_clkpm_nocheck()
153 	link->clkpm_enabled = !!enable;  in pcie_set_clkpm_nocheck()
156 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)  in pcie_set_clkpm()  argument
159 	 * Don't enable Clock PM if the link is not Clock PM capable  in pcie_set_clkpm()
162 	if (!link->clkpm_capable || link->clkpm_disable)  in pcie_set_clkpm()
165 	if (link->clkpm_enabled == enable)  in pcie_set_clkpm()
167 	pcie_set_clkpm_nocheck(link, enable);  in pcie_set_clkpm()
170 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_clkpm_cap_init()  argument
176 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_clkpm_cap_init()
190 	link->clkpm_enabled = enabled;  in pcie_clkpm_cap_init()
191 	link->clkpm_default = enabled;  in pcie_clkpm_cap_init()
192 	link->clkpm_capable = capable;  in pcie_clkpm_cap_init()
193 	link->clkpm_disable = blacklist ? 1 : 0;  in pcie_clkpm_cap_init()
196 static bool pcie_retrain_link(struct pcie_link_state *link)  in pcie_retrain_link()  argument
198 	struct pci_dev *parent = link->pdev;  in pcie_retrain_link()
207 		 * Due to an erratum in some devices the Retrain Link bit  in pcie_retrain_link()
208 		 * needs to be cleared again manually to allow the link  in pcie_retrain_link()
215 	/* Wait for link training end. Break out after waiting for timeout */  in pcie_retrain_link()
227  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
231 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)  in pcie_aspm_configure_common_clock()  argument
235 	struct pci_dev *child, *parent = link->pdev;  in pcie_aspm_configure_common_clock()
292 	if (pcie_retrain_link(link))  in pcie_aspm_configure_common_clock()
400 	struct pcie_link_state *link;  in pcie_aspm_check_latency()  local
407 	link = endpoint->bus->self->link_state;  in pcie_aspm_check_latency()
417 	while (link) {  in pcie_aspm_check_latency()
418 		struct pci_dev *dev = pci_function_0(link->pdev->subordinate);  in pcie_aspm_check_latency()
421 		pcie_capability_read_dword(link->pdev, PCI_EXP_LNKCAP,  in pcie_aspm_check_latency()
431 		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&  in pcie_aspm_check_latency()
433 			link->aspm_capable &= ~ASPM_STATE_L0S_UP;  in pcie_aspm_check_latency()
436 		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&  in pcie_aspm_check_latency()
438 			link->aspm_capable &= ~ASPM_STATE_L0S_DW;  in pcie_aspm_check_latency()
453 		if ((link->aspm_capable & ASPM_STATE_L1) &&  in pcie_aspm_check_latency()
455 			link->aspm_capable &= ~ASPM_STATE_L1;  in pcie_aspm_check_latency()
458 		link = link->parent;  in pcie_aspm_check_latency()
499 static void aspm_calc_l1ss_info(struct pcie_link_state *link,  in aspm_calc_l1ss_info()  argument
502 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_calc_l1ss_info()
508 	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))  in aspm_calc_l1ss_info()
533 	 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if  in aspm_calc_l1ss_info()
572 static void aspm_l1ss_init(struct pcie_link_state *link)  in aspm_l1ss_init()  argument
574 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_l1ss_init()
601 		link->aspm_support |= ASPM_STATE_L1_1;  in aspm_l1ss_init()
603 		link->aspm_support |= ASPM_STATE_L1_2;  in aspm_l1ss_init()
605 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
607 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
617 		link->aspm_enabled |= ASPM_STATE_L1_1;  in aspm_l1ss_init()
619 		link->aspm_enabled |= ASPM_STATE_L1_2;  in aspm_l1ss_init()
621 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;  in aspm_l1ss_init()
623 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;  in aspm_l1ss_init()
625 	if (link->aspm_support & ASPM_STATE_L1SS)  in aspm_l1ss_init()
626 		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);  in aspm_l1ss_init()
629 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_aspm_cap_init()  argument
631 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_aspm_cap_init()
638 		link->aspm_enabled = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
639 		link->aspm_disable = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
644 	 * If ASPM not supported, don't mess with the clocks and link,  in pcie_aspm_cap_init()
653 	pcie_aspm_configure_common_clock(link);  in pcie_aspm_cap_init()
658 	 * read-only Link Capabilities may change depending on common clock  in pcie_aspm_cap_init()
670 	 * given link unless components on both sides of the link each  in pcie_aspm_cap_init()
674 		link->aspm_support |= ASPM_STATE_L0S;  in pcie_aspm_cap_init()
677 		link->aspm_enabled |= ASPM_STATE_L0S_UP;  in pcie_aspm_cap_init()
679 		link->aspm_enabled |= ASPM_STATE_L0S_DW;  in pcie_aspm_cap_init()
683 		link->aspm_support |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
686 		link->aspm_enabled |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
688 	aspm_l1ss_init(link);  in pcie_aspm_cap_init()
691 	link->aspm_default = link->aspm_enabled;  in pcie_aspm_cap_init()
694 	link->aspm_capable = link->aspm_support;  in pcie_aspm_cap_init()
707 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_l1ss()  argument
710 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_l1ss()
712 	enable_req = (link->aspm_enabled ^ state) & state;  in pcie_config_aspm_l1ss()
803 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_link()  argument
806 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_link()
810 	state &= (link->aspm_capable & ~link->aspm_disable);  in pcie_config_aspm_link()
819 		state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);  in pcie_config_aspm_link()
822 	/* Nothing to do if the link is already in the requested state */  in pcie_config_aspm_link()
823 	if (link->aspm_enabled == state)  in pcie_config_aspm_link()
835 	if (link->aspm_capable & ASPM_STATE_L1SS)  in pcie_config_aspm_link()
836 		pcie_config_aspm_l1ss(link, state);  in pcie_config_aspm_link()
851 	link->aspm_enabled = state;  in pcie_config_aspm_link()
854 static void pcie_config_aspm_path(struct pcie_link_state *link)  in pcie_config_aspm_path()  argument
856 	while (link) {  in pcie_config_aspm_path()
857 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_config_aspm_path()
858 		link = link->parent;  in pcie_config_aspm_path()
862 static void free_link_state(struct pcie_link_state *link)  in free_link_state()  argument
864 	link->pdev->link_state = NULL;  in free_link_state()
865 	kfree(link);  in free_link_state()
905 	struct pcie_link_state *link;  in alloc_pcie_link_state()  local
907 	link = kzalloc(sizeof(*link), GFP_KERNEL);  in alloc_pcie_link_state()
908 	if (!link)  in alloc_pcie_link_state()
911 	INIT_LIST_HEAD(&link->sibling);  in alloc_pcie_link_state()
912 	link->pdev = pdev;  in alloc_pcie_link_state()
913 	link->downstream = pci_function_0(pdev->subordinate);  in alloc_pcie_link_state()
919 	 * a switch may become the root of the link state chain for all  in alloc_pcie_link_state()
925 		link->root = link;  in alloc_pcie_link_state()
931 			kfree(link);  in alloc_pcie_link_state()
935 		link->parent = parent;  in alloc_pcie_link_state()
936 		link->root = link->parent->root;  in alloc_pcie_link_state()
939 	list_add(&link->sibling, &link_list);  in alloc_pcie_link_state()
940 	pdev->link_state = link;  in alloc_pcie_link_state()
941 	return link;  in alloc_pcie_link_state()
953  * pcie_aspm_init_link_state: Initiate PCI express link state.
959 	struct pcie_link_state *link;  in pcie_aspm_init_link_state()  local
970 	 * end of a Link, so there's nothing to do unless this device is  in pcie_aspm_init_link_state()
986 	link = alloc_pcie_link_state(pdev);  in pcie_aspm_init_link_state()
987 	if (!link)  in pcie_aspm_init_link_state()
994 	pcie_aspm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
997 	pcie_clkpm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
1001 	 * link policy setting. Enabling ASPM on broken hardware can cripple  in pcie_aspm_init_link_state()
1009 		pcie_config_aspm_path(link);  in pcie_aspm_init_link_state()
1010 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_init_link_state()
1024 	struct pcie_link_state *link;  in pcie_update_aspm_capable()  local
1026 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
1027 		if (link->root != root)  in pcie_update_aspm_capable()
1029 		link->aspm_capable = link->aspm_support;  in pcie_update_aspm_capable()
1031 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
1033 		struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_update_aspm_capable()
1034 		if (link->root != root)  in pcie_update_aspm_capable()
1049 	struct pcie_link_state *link, *root, *parent_link;  in pcie_aspm_exit_link_state()  local
1063 	link = parent->link_state;  in pcie_aspm_exit_link_state()
1064 	root = link->root;  in pcie_aspm_exit_link_state()
1065 	parent_link = link->parent;  in pcie_aspm_exit_link_state()
1067 	/* All functions are removed, so just disable ASPM for the link */  in pcie_aspm_exit_link_state()
1068 	pcie_config_aspm_link(link, 0);  in pcie_aspm_exit_link_state()
1069 	list_del(&link->sibling);  in pcie_aspm_exit_link_state()
1071 	free_link_state(link);  in pcie_aspm_exit_link_state()
1085 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_powersave_config_link()  local
1087 	if (aspm_disabled || !link)  in pcie_aspm_powersave_config_link()
1096 	pcie_config_aspm_path(link);  in pcie_aspm_powersave_config_link()
1097 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_powersave_config_link()
1118 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_disable_link_state()  local
1120 	if (!link)  in __pci_disable_link_state()
1139 		link->aspm_disable |= ASPM_STATE_L0S;  in __pci_disable_link_state()
1142 		link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;  in __pci_disable_link_state()
1144 		link->aspm_disable |= ASPM_STATE_L1_1;  in __pci_disable_link_state()
1146 		link->aspm_disable |= ASPM_STATE_L1_2;  in __pci_disable_link_state()
1148 		link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;  in __pci_disable_link_state()
1150 		link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;  in __pci_disable_link_state()
1151 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_disable_link_state()
1154 		link->clkpm_disable = 1;  in __pci_disable_link_state()
1155 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_disable_link_state()
1170  * pci_disable_link_state - Disable device's link state, so the link will
1176  * @state: ASPM link state to disable
1188 	struct pcie_link_state *link;  in pcie_aspm_set_policy()  local
1201 	list_for_each_entry(link, &link_list, sibling) {  in pcie_aspm_set_policy()
1202 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_aspm_set_policy()
1203 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_set_policy()
1236 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in pcie_aspm_enabled()  local
1238 	if (!link)  in pcie_aspm_enabled()
1241 	return link->aspm_enabled;  in pcie_aspm_enabled()
1250 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_show_common()  local
1252 	return sysfs_emit(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);  in aspm_attr_show_common()
1260 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_store_common()  local
1270 		link->aspm_disable &= ~state;  in aspm_attr_store_common()
1273 			link->aspm_disable &= ~ASPM_STATE_L1;  in aspm_attr_store_common()
1275 		link->aspm_disable |= state;  in aspm_attr_store_common()
1278 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in aspm_attr_store_common()
1307 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in ASPM_ATTR()  local
1309 	return sysfs_emit(buf, "%d\n", link->clkpm_enabled);  in ASPM_ATTR()
1317 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in clkpm_store()  local
1326 	link->clkpm_disable = !state_enable;  in clkpm_store()
1327 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in clkpm_store()
1359 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_ctrl_attrs_are_visible()  local
1369 	if (aspm_disabled || !link)  in aspm_ctrl_attrs_are_visible()
1373 		return link->clkpm_capable ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1375 	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1379 	.name = "link",