3  * Enable PCIe link L0s/L1 state and Clock Power Management
50 	struct pci_dev *pdev;		/* Upstream component of the Link */
52 	struct pcie_link_state *root;	/* pointer to the root port link */
53 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
108 static int policy_to_aspm_state(struct pcie_link_state *link)  in policy_to_aspm_state()  argument
121 		return link->aspm_default;  in policy_to_aspm_state()
126 static int policy_to_clkpm_state(struct pcie_link_state *link)  in policy_to_clkpm_state()  argument
137 		return link->clkpm_default;  in policy_to_clkpm_state()
142 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)  in pcie_set_clkpm_nocheck()  argument
145 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_set_clkpm_nocheck()
152 	link->clkpm_enabled = !!enable;  in pcie_set_clkpm_nocheck()
155 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)  in pcie_set_clkpm()  argument
158 	 * Don't enable Clock PM if the link is not Clock PM capable  in pcie_set_clkpm()
161 	if (!link->clkpm_capable || link->clkpm_disable)  in pcie_set_clkpm()
164 	if (link->clkpm_enabled == enable)  in pcie_set_clkpm()
166 	pcie_set_clkpm_nocheck(link, enable);  in pcie_set_clkpm()
169 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_clkpm_cap_init()  argument
175 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_clkpm_cap_init()
189 	link->clkpm_enabled = enabled;  in pcie_clkpm_cap_init()
190 	link->clkpm_default = enabled;  in pcie_clkpm_cap_init()
191 	link->clkpm_capable = capable;  in pcie_clkpm_cap_init()
192 	link->clkpm_disable = blacklist ? 1 : 0;  in pcie_clkpm_cap_init()
195 static bool pcie_retrain_link(struct pcie_link_state *link)  in pcie_retrain_link()  argument
197 	struct pci_dev *parent = link->pdev;  in pcie_retrain_link()
206 		 * Due to an erratum in some devices the Retrain Link bit  in pcie_retrain_link()
207 		 * needs to be cleared again manually to allow the link  in pcie_retrain_link()
214 	/* Wait for link training end. Break out after waiting for timeout */  in pcie_retrain_link()
226  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
230 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)  in pcie_aspm_configure_common_clock()  argument
234 	struct pci_dev *child, *parent = link->pdev;  in pcie_aspm_configure_common_clock()
291 	if (pcie_retrain_link(link))  in pcie_aspm_configure_common_clock()
383 	struct pcie_link_state *link;  in pcie_aspm_check_latency()  local
390 	link = endpoint->bus->self->link_state;  in pcie_aspm_check_latency()
391 	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];  in pcie_aspm_check_latency()
393 	while (link) {  in pcie_aspm_check_latency()
395 		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&  in pcie_aspm_check_latency()
396 		    (link->latency_up.l0s > acceptable->l0s))  in pcie_aspm_check_latency()
397 			link->aspm_capable &= ~ASPM_STATE_L0S_UP;  in pcie_aspm_check_latency()
400 		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&  in pcie_aspm_check_latency()
401 		    (link->latency_dw.l0s > acceptable->l0s))  in pcie_aspm_check_latency()
402 			link->aspm_capable &= ~ASPM_STATE_L0S_DW;  in pcie_aspm_check_latency()
416 		latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);  in pcie_aspm_check_latency()
417 		if ((link->aspm_capable & ASPM_STATE_L1) &&  in pcie_aspm_check_latency()
419 			link->aspm_capable &= ~ASPM_STATE_L1;  in pcie_aspm_check_latency()
422 		link = link->parent;  in pcie_aspm_check_latency()
452 static void aspm_calc_l1ss_info(struct pcie_link_state *link,  in aspm_calc_l1ss_info()  argument
455 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in aspm_calc_l1ss_info()
462 	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))  in aspm_calc_l1ss_info()
487 	 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if  in aspm_calc_l1ss_info()
543 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_aspm_cap_init()  argument
545 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_aspm_cap_init()
554 		link->aspm_enabled = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
555 		link->aspm_disable = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
560 	 * If ASPM not supported, don't mess with the clocks and link,  in pcie_aspm_cap_init()
569 	pcie_aspm_configure_common_clock(link);  in pcie_aspm_cap_init()
574 	 * read-only Link Capabilities may change depending on common clock  in pcie_aspm_cap_init()
586 	 * given link unless components on both sides of the link each  in pcie_aspm_cap_init()
590 		link->aspm_support |= ASPM_STATE_L0S;  in pcie_aspm_cap_init()
593 		link->aspm_enabled |= ASPM_STATE_L0S_UP;  in pcie_aspm_cap_init()
595 		link->aspm_enabled |= ASPM_STATE_L0S_DW;  in pcie_aspm_cap_init()
596 	link->latency_up.l0s = calc_l0s_latency(parent_lnkcap);  in pcie_aspm_cap_init()
597 	link->latency_dw.l0s = calc_l0s_latency(child_lnkcap);  in pcie_aspm_cap_init()
601 		link->aspm_support |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
604 		link->aspm_enabled |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
605 	link->latency_up.l1 = calc_l1_latency(parent_lnkcap);  in pcie_aspm_cap_init()
606 	link->latency_dw.l1 = calc_l1_latency(child_lnkcap);  in pcie_aspm_cap_init()
628 		link->aspm_support |= ASPM_STATE_L1_1;  in pcie_aspm_cap_init()
630 		link->aspm_support |= ASPM_STATE_L1_2;  in pcie_aspm_cap_init()
632 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;  in pcie_aspm_cap_init()
634 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;  in pcie_aspm_cap_init()
644 		link->aspm_enabled |= ASPM_STATE_L1_1;  in pcie_aspm_cap_init()
646 		link->aspm_enabled |= ASPM_STATE_L1_2;  in pcie_aspm_cap_init()
648 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;  in pcie_aspm_cap_init()
650 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;  in pcie_aspm_cap_init()
652 	if (link->aspm_support & ASPM_STATE_L1SS)  in pcie_aspm_cap_init()
653 		aspm_calc_l1ss_info(link, parent_l1ss_cap, child_l1ss_cap);  in pcie_aspm_cap_init()
656 	link->aspm_default = link->aspm_enabled;  in pcie_aspm_cap_init()
659 	link->aspm_capable = link->aspm_support;  in pcie_aspm_cap_init()
665 			&link->acceptable[PCI_FUNC(child->devfn)];  in pcie_aspm_cap_init()
684 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_l1ss()  argument
687 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_l1ss()
689 	enable_req = (link->aspm_enabled ^ state) & state;  in pcie_config_aspm_l1ss()
743 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_link()  argument
746 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_link()
750 	state &= (link->aspm_capable & ~link->aspm_disable);  in pcie_config_aspm_link()
759 		state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);  in pcie_config_aspm_link()
762 	/* Nothing to do if the link is already in the requested state */  in pcie_config_aspm_link()
763 	if (link->aspm_enabled == state)  in pcie_config_aspm_link()
775 	if (link->aspm_capable & ASPM_STATE_L1SS)  in pcie_config_aspm_link()
776 		pcie_config_aspm_l1ss(link, state);  in pcie_config_aspm_link()
791 	link->aspm_enabled = state;  in pcie_config_aspm_link()
794 static void pcie_config_aspm_path(struct pcie_link_state *link)  in pcie_config_aspm_path()  argument
796 	while (link) {  in pcie_config_aspm_path()
797 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_config_aspm_path()
798 		link = link->parent;  in pcie_config_aspm_path()
802 static void free_link_state(struct pcie_link_state *link)  in free_link_state()  argument
804 	link->pdev->link_state = NULL;  in free_link_state()
805 	kfree(link);  in free_link_state()
845 	struct pcie_link_state *link;  in alloc_pcie_link_state()  local
847 	link = kzalloc(sizeof(*link), GFP_KERNEL);  in alloc_pcie_link_state()
848 	if (!link)  in alloc_pcie_link_state()
851 	INIT_LIST_HEAD(&link->sibling);  in alloc_pcie_link_state()
852 	link->pdev = pdev;  in alloc_pcie_link_state()
853 	link->downstream = pci_function_0(pdev->subordinate);  in alloc_pcie_link_state()
859 	 * a switch may become the root of the link state chain for all  in alloc_pcie_link_state()
865 		link->root = link;  in alloc_pcie_link_state()
871 			kfree(link);  in alloc_pcie_link_state()
875 		link->parent = parent;  in alloc_pcie_link_state()
876 		link->root = link->parent->root;  in alloc_pcie_link_state()
879 	list_add(&link->sibling, &link_list);  in alloc_pcie_link_state()
880 	pdev->link_state = link;  in alloc_pcie_link_state()
881 	return link;  in alloc_pcie_link_state()
893  * pcie_aspm_init_link_state: Initiate PCI express link state.
899 	struct pcie_link_state *link;  in pcie_aspm_init_link_state()  local
910 	 * end of a Link, so there's nothing to do unless this device is  in pcie_aspm_init_link_state()
926 	link = alloc_pcie_link_state(pdev);  in pcie_aspm_init_link_state()
927 	if (!link)  in pcie_aspm_init_link_state()
934 	pcie_aspm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
937 	pcie_clkpm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
941 	 * link policy setting. Enabling ASPM on broken hardware can cripple  in pcie_aspm_init_link_state()
949 		pcie_config_aspm_path(link);  in pcie_aspm_init_link_state()
950 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_init_link_state()
964 	struct pcie_link_state *link;  in pcie_update_aspm_capable()  local
966 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
967 		if (link->root != root)  in pcie_update_aspm_capable()
969 		link->aspm_capable = link->aspm_support;  in pcie_update_aspm_capable()
971 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
973 		struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_update_aspm_capable()
974 		if (link->root != root)  in pcie_update_aspm_capable()
989 	struct pcie_link_state *link, *root, *parent_link;  in pcie_aspm_exit_link_state()  local
1003 	link = parent->link_state;  in pcie_aspm_exit_link_state()
1004 	root = link->root;  in pcie_aspm_exit_link_state()
1005 	parent_link = link->parent;  in pcie_aspm_exit_link_state()
1007 	/* All functions are removed, so just disable ASPM for the link */  in pcie_aspm_exit_link_state()
1008 	pcie_config_aspm_link(link, 0);  in pcie_aspm_exit_link_state()
1009 	list_del(&link->sibling);  in pcie_aspm_exit_link_state()
1011 	free_link_state(link);  in pcie_aspm_exit_link_state()
1026 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_pm_state_change()  local
1028 	if (aspm_disabled || !link)  in pcie_aspm_pm_state_change()
1036 	pcie_update_aspm_capable(link->root);  in pcie_aspm_pm_state_change()
1037 	pcie_config_aspm_path(link);  in pcie_aspm_pm_state_change()
1044 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_powersave_config_link()  local
1046 	if (aspm_disabled || !link)  in pcie_aspm_powersave_config_link()
1055 	pcie_config_aspm_path(link);  in pcie_aspm_powersave_config_link()
1056 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_powersave_config_link()
1077 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in __pci_disable_link_state()  local
1079 	if (!link)  in __pci_disable_link_state()
1098 		link->aspm_disable |= ASPM_STATE_L0S;  in __pci_disable_link_state()
1101 		link->aspm_disable |= ASPM_STATE_L1 | ASPM_STATE_L1SS;  in __pci_disable_link_state()
1103 		link->aspm_disable |= ASPM_STATE_L1_1;  in __pci_disable_link_state()
1105 		link->aspm_disable |= ASPM_STATE_L1_2;  in __pci_disable_link_state()
1107 		link->aspm_disable |= ASPM_STATE_L1_1_PCIPM;  in __pci_disable_link_state()
1109 		link->aspm_disable |= ASPM_STATE_L1_2_PCIPM;  in __pci_disable_link_state()
1110 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_disable_link_state()
1113 		link->clkpm_disable = 1;  in __pci_disable_link_state()
1114 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in __pci_disable_link_state()
1129  * pci_disable_link_state - Disable device's link state, so the link will
1135  * @state: ASPM link state to disable
1147 	struct pcie_link_state *link;  in pcie_aspm_set_policy()  local
1160 	list_for_each_entry(link, &link_list, sibling) {  in pcie_aspm_set_policy()
1161 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_aspm_set_policy()
1162 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_set_policy()
1195 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in pcie_aspm_enabled()  local
1197 	if (!link)  in pcie_aspm_enabled()
1200 	return link->aspm_enabled;  in pcie_aspm_enabled()
1209 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_show_common()  local
1211 	return sprintf(buf, "%d\n", (link->aspm_enabled & state) ? 1 : 0);  in aspm_attr_show_common()
1219 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_attr_store_common()  local
1229 		link->aspm_disable &= ~state;  in aspm_attr_store_common()
1232 			link->aspm_disable &= ~ASPM_STATE_L1;  in aspm_attr_store_common()
1234 		link->aspm_disable |= state;  in aspm_attr_store_common()
1237 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in aspm_attr_store_common()
1266 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in ASPM_ATTR()  local
1268 	return sprintf(buf, "%d\n", link->clkpm_enabled);  in ASPM_ATTR()
1276 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in clkpm_store()  local
1285 	link->clkpm_disable = !state_enable;  in clkpm_store()
1286 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in clkpm_store()
1318 	struct pcie_link_state *link = pcie_aspm_get_link(pdev);  in aspm_ctrl_attrs_are_visible()  local
1328 	if (aspm_disabled || !link)  in aspm_ctrl_attrs_are_visible()
1332 		return link->clkpm_capable ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1334 	return link->aspm_capable & aspm_state_map[n - 1] ? a->mode : 0;  in aspm_ctrl_attrs_are_visible()
1338 	.name = "link",