3  * Enable PCIe link L0s/L1 state and Clock Power Management
50 	struct pci_dev *pdev;		/* Upstream component of the Link */
52 	struct pcie_link_state *root;	/* pointer to the root port link */
53 	struct pcie_link_state *parent;	/* pointer to the parent Link state */
115 static int policy_to_aspm_state(struct pcie_link_state *link)  in policy_to_aspm_state()  argument
128 		return link->aspm_default;  in policy_to_aspm_state()
133 static int policy_to_clkpm_state(struct pcie_link_state *link)  in policy_to_clkpm_state()  argument
144 		return link->clkpm_default;  in policy_to_clkpm_state()
149 static void pcie_set_clkpm_nocheck(struct pcie_link_state *link, int enable)  in pcie_set_clkpm_nocheck()  argument
152 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_set_clkpm_nocheck()
159 	link->clkpm_enabled = !!enable;  in pcie_set_clkpm_nocheck()
162 static void pcie_set_clkpm(struct pcie_link_state *link, int enable)  in pcie_set_clkpm()  argument
164 	/* Don't enable Clock PM if the link is not Clock PM capable */  in pcie_set_clkpm()
165 	if (!link->clkpm_capable)  in pcie_set_clkpm()
168 	if (link->clkpm_enabled == enable)  in pcie_set_clkpm()
170 	pcie_set_clkpm_nocheck(link, enable);  in pcie_set_clkpm()
173 static void pcie_clkpm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_clkpm_cap_init()  argument
179 	struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_clkpm_cap_init()
193 	link->clkpm_enabled = enabled;  in pcie_clkpm_cap_init()
194 	link->clkpm_default = enabled;  in pcie_clkpm_cap_init()
195 	link->clkpm_capable = (blacklist) ? 0 : capable;  in pcie_clkpm_cap_init()
198 static bool pcie_retrain_link(struct pcie_link_state *link)  in pcie_retrain_link()  argument
200 	struct pci_dev *parent = link->pdev;  in pcie_retrain_link()
209 		 * Due to an erratum in some devices the Retrain Link bit  in pcie_retrain_link()
210 		 * needs to be cleared again manually to allow the link  in pcie_retrain_link()
217 	/* Wait for link training end. Break out after waiting for timeout */  in pcie_retrain_link()
229  * pcie_aspm_configure_common_clock: check if the 2 ends of a link
233 static void pcie_aspm_configure_common_clock(struct pcie_link_state *link)  in pcie_aspm_configure_common_clock()  argument
237 	struct pci_dev *child, *parent = link->pdev;  in pcie_aspm_configure_common_clock()
294 	if (pcie_retrain_link(link))  in pcie_aspm_configure_common_clock()
434 	struct pcie_link_state *link;  in pcie_aspm_check_latency()  local
441 	link = endpoint->bus->self->link_state;  in pcie_aspm_check_latency()
442 	acceptable = &link->acceptable[PCI_FUNC(endpoint->devfn)];  in pcie_aspm_check_latency()
444 	while (link) {  in pcie_aspm_check_latency()
446 		if ((link->aspm_capable & ASPM_STATE_L0S_UP) &&  in pcie_aspm_check_latency()
447 		    (link->latency_up.l0s > acceptable->l0s))  in pcie_aspm_check_latency()
448 			link->aspm_capable &= ~ASPM_STATE_L0S_UP;  in pcie_aspm_check_latency()
451 		if ((link->aspm_capable & ASPM_STATE_L0S_DW) &&  in pcie_aspm_check_latency()
452 		    (link->latency_dw.l0s > acceptable->l0s))  in pcie_aspm_check_latency()
453 			link->aspm_capable &= ~ASPM_STATE_L0S_DW;  in pcie_aspm_check_latency()
467 		latency = max_t(u32, link->latency_up.l1, link->latency_dw.l1);  in pcie_aspm_check_latency()
468 		if ((link->aspm_capable & ASPM_STATE_L1) &&  in pcie_aspm_check_latency()
470 			link->aspm_capable &= ~ASPM_STATE_L1;  in pcie_aspm_check_latency()
473 		link = link->parent;  in pcie_aspm_check_latency()
492 static void aspm_calc_l1ss_info(struct pcie_link_state *link,  in aspm_calc_l1ss_info()  argument
499 	link->l1ss.up_cap_ptr = upreg->l1ss_cap_ptr;  in aspm_calc_l1ss_info()
500 	link->l1ss.dw_cap_ptr = dwreg->l1ss_cap_ptr;  in aspm_calc_l1ss_info()
501 	link->l1ss.ctl1 = link->l1ss.ctl2 = 0;  in aspm_calc_l1ss_info()
503 	if (!(link->aspm_support & ASPM_STATE_L1_2_MASK))  in aspm_calc_l1ss_info()
517 	if (calc_l1ss_pwron(link->pdev, scale1, val1) >  in aspm_calc_l1ss_info()
518 	    calc_l1ss_pwron(link->downstream, scale2, val2)) {  in aspm_calc_l1ss_info()
519 		link->l1ss.ctl2 |= scale1 | (val1 << 3);  in aspm_calc_l1ss_info()
520 		t_power_on = calc_l1ss_pwron(link->pdev, scale1, val1);  in aspm_calc_l1ss_info()
522 		link->l1ss.ctl2 |= scale2 | (val2 << 3);  in aspm_calc_l1ss_info()
523 		t_power_on = calc_l1ss_pwron(link->downstream, scale2, val2);  in aspm_calc_l1ss_info()
528 	 * Link from L0 to L1.2 and back to L0 so we enter L1.2 only if  in aspm_calc_l1ss_info()
538 	link->l1ss.ctl1 |= t_common_mode << 8 | scale << 29 | value << 16;  in aspm_calc_l1ss_info()
541 static void pcie_aspm_cap_init(struct pcie_link_state *link, int blacklist)  in pcie_aspm_cap_init()  argument
543 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_aspm_cap_init()
549 		link->aspm_enabled = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
550 		link->aspm_disable = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
559 	 * If ASPM not supported, don't mess with the clocks and link,  in pcie_aspm_cap_init()
566 	pcie_aspm_configure_common_clock(link);  in pcie_aspm_cap_init()
579 	 * given link unless components on both sides of the link each  in pcie_aspm_cap_init()
583 		link->aspm_support |= ASPM_STATE_L0S;  in pcie_aspm_cap_init()
585 		link->aspm_enabled |= ASPM_STATE_L0S_UP;  in pcie_aspm_cap_init()
587 		link->aspm_enabled |= ASPM_STATE_L0S_DW;  in pcie_aspm_cap_init()
588 	link->latency_up.l0s = calc_l0s_latency(upreg.latency_encoding_l0s);  in pcie_aspm_cap_init()
589 	link->latency_dw.l0s = calc_l0s_latency(dwreg.latency_encoding_l0s);  in pcie_aspm_cap_init()
593 		link->aspm_support |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
595 		link->aspm_enabled |= ASPM_STATE_L1;  in pcie_aspm_cap_init()
596 	link->latency_up.l1 = calc_l1_latency(upreg.latency_encoding_l1);  in pcie_aspm_cap_init()
597 	link->latency_dw.l1 = calc_l1_latency(dwreg.latency_encoding_l1);  in pcie_aspm_cap_init()
601 		link->aspm_support |= ASPM_STATE_L1_1;  in pcie_aspm_cap_init()
603 		link->aspm_support |= ASPM_STATE_L1_2;  in pcie_aspm_cap_init()
605 		link->aspm_support |= ASPM_STATE_L1_1_PCIPM;  in pcie_aspm_cap_init()
607 		link->aspm_support |= ASPM_STATE_L1_2_PCIPM;  in pcie_aspm_cap_init()
610 		link->aspm_enabled |= ASPM_STATE_L1_1;  in pcie_aspm_cap_init()
612 		link->aspm_enabled |= ASPM_STATE_L1_2;  in pcie_aspm_cap_init()
614 		link->aspm_enabled |= ASPM_STATE_L1_1_PCIPM;  in pcie_aspm_cap_init()
616 		link->aspm_enabled |= ASPM_STATE_L1_2_PCIPM;  in pcie_aspm_cap_init()
618 	if (link->aspm_support & ASPM_STATE_L1SS)  in pcie_aspm_cap_init()
619 		aspm_calc_l1ss_info(link, &upreg, &dwreg);  in pcie_aspm_cap_init()
622 	link->aspm_default = link->aspm_enabled;  in pcie_aspm_cap_init()
625 	link->aspm_capable = link->aspm_support;  in pcie_aspm_cap_init()
632 			link->aspm_disable = ASPM_STATE_ALL;  in pcie_aspm_cap_init()
641 			&link->acceptable[PCI_FUNC(child->devfn)];  in pcie_aspm_cap_init()
671 static void pcie_config_aspm_l1ss(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_l1ss()  argument
674 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_l1ss()
675 	u32 up_cap_ptr = link->l1ss.up_cap_ptr;  in pcie_config_aspm_l1ss()
676 	u32 dw_cap_ptr = link->l1ss.dw_cap_ptr;  in pcie_config_aspm_l1ss()
678 	enable_req = (link->aspm_enabled ^ state) & state;  in pcie_config_aspm_l1ss()
713 				       link->l1ss.ctl2);  in pcie_config_aspm_l1ss()
715 				       link->l1ss.ctl2);  in pcie_config_aspm_l1ss()
720 					link->l1ss.ctl1);  in pcie_config_aspm_l1ss()
726 					link->l1ss.ctl1);  in pcie_config_aspm_l1ss()
730 					link->l1ss.ctl1);  in pcie_config_aspm_l1ss()
756 static void pcie_config_aspm_link(struct pcie_link_state *link, u32 state)  in pcie_config_aspm_link()  argument
759 	struct pci_dev *child = link->downstream, *parent = link->pdev;  in pcie_config_aspm_link()
763 	state &= (link->aspm_capable & ~link->aspm_disable);  in pcie_config_aspm_link()
772 		state |= (link->aspm_enabled & ASPM_STATE_L1_SS_PCIPM);  in pcie_config_aspm_link()
775 	/* Nothing to do if the link is already in the requested state */  in pcie_config_aspm_link()
776 	if (link->aspm_enabled == state)  in pcie_config_aspm_link()
788 	if (link->aspm_capable & ASPM_STATE_L1SS)  in pcie_config_aspm_link()
789 		pcie_config_aspm_l1ss(link, state);  in pcie_config_aspm_link()
804 	link->aspm_enabled = state;  in pcie_config_aspm_link()
807 static void pcie_config_aspm_path(struct pcie_link_state *link)  in pcie_config_aspm_path()  argument
809 	while (link) {  in pcie_config_aspm_path()
810 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_config_aspm_path()
811 		link = link->parent;  in pcie_config_aspm_path()
815 static void free_link_state(struct pcie_link_state *link)  in free_link_state()  argument
817 	link->pdev->link_state = NULL;  in free_link_state()
818 	kfree(link);  in free_link_state()
858 	struct pcie_link_state *link;  in alloc_pcie_link_state()  local
860 	link = kzalloc(sizeof(*link), GFP_KERNEL);  in alloc_pcie_link_state()
861 	if (!link)  in alloc_pcie_link_state()
864 	INIT_LIST_HEAD(&link->sibling);  in alloc_pcie_link_state()
865 	link->pdev = pdev;  in alloc_pcie_link_state()
866 	link->downstream = pci_function_0(pdev->subordinate);  in alloc_pcie_link_state()
872 	 * a switch may become the root of the link state chain for all  in alloc_pcie_link_state()
878 		link->root = link;  in alloc_pcie_link_state()
884 			kfree(link);  in alloc_pcie_link_state()
888 		link->parent = parent;  in alloc_pcie_link_state()
889 		link->root = link->parent->root;  in alloc_pcie_link_state()
892 	list_add(&link->sibling, &link_list);  in alloc_pcie_link_state()
893 	pdev->link_state = link;  in alloc_pcie_link_state()
894 	return link;  in alloc_pcie_link_state()
898  * pcie_aspm_init_link_state: Initiate PCI express link state.
904 	struct pcie_link_state *link;  in pcie_aspm_init_link_state()  local
915 	 * end of a Link, so there's nothing to do unless this device is  in pcie_aspm_init_link_state()
931 	link = alloc_pcie_link_state(pdev);  in pcie_aspm_init_link_state()
932 	if (!link)  in pcie_aspm_init_link_state()
939 	pcie_aspm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
942 	pcie_clkpm_cap_init(link, blacklist);  in pcie_aspm_init_link_state()
946 	 * link policy setting. Enabling ASPM on broken hardware can cripple  in pcie_aspm_init_link_state()
954 		pcie_config_aspm_path(link);  in pcie_aspm_init_link_state()
955 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_init_link_state()
967 	struct pcie_link_state *link;  in pcie_update_aspm_capable()  local
969 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
970 		if (link->root != root)  in pcie_update_aspm_capable()
972 		link->aspm_capable = link->aspm_support;  in pcie_update_aspm_capable()
974 	list_for_each_entry(link, &link_list, sibling) {  in pcie_update_aspm_capable()
976 		struct pci_bus *linkbus = link->pdev->subordinate;  in pcie_update_aspm_capable()
977 		if (link->root != root)  in pcie_update_aspm_capable()
992 	struct pcie_link_state *link, *root, *parent_link;  in pcie_aspm_exit_link_state()  local
1006 	link = parent->link_state;  in pcie_aspm_exit_link_state()
1007 	root = link->root;  in pcie_aspm_exit_link_state()
1008 	parent_link = link->parent;  in pcie_aspm_exit_link_state()
1010 	/* All functions are removed, so just disable ASPM for the link */  in pcie_aspm_exit_link_state()
1011 	pcie_config_aspm_link(link, 0);  in pcie_aspm_exit_link_state()
1012 	list_del(&link->sibling);  in pcie_aspm_exit_link_state()
1014 	free_link_state(link);  in pcie_aspm_exit_link_state()
1029 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_pm_state_change()  local
1031 	if (aspm_disabled || !link)  in pcie_aspm_pm_state_change()
1039 	pcie_update_aspm_capable(link->root);  in pcie_aspm_pm_state_change()
1040 	pcie_config_aspm_path(link);  in pcie_aspm_pm_state_change()
1047 	struct pcie_link_state *link = pdev->link_state;  in pcie_aspm_powersave_config_link()  local
1049 	if (aspm_disabled || !link)  in pcie_aspm_powersave_config_link()
1058 	pcie_config_aspm_path(link);  in pcie_aspm_powersave_config_link()
1059 	pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_powersave_config_link()
1067 	struct pcie_link_state *link;  in __pci_disable_link_state()  local
1093 	link = parent->link_state;  in __pci_disable_link_state()
1095 		link->aspm_disable |= ASPM_STATE_L0S;  in __pci_disable_link_state()
1097 		link->aspm_disable |= ASPM_STATE_L1;  in __pci_disable_link_state()
1098 	pcie_config_aspm_link(link, policy_to_aspm_state(link));  in __pci_disable_link_state()
1101 		link->clkpm_capable = 0;  in __pci_disable_link_state()
1102 		pcie_set_clkpm(link, 0);  in __pci_disable_link_state()
1118  * pci_disable_link_state - Disable device's link state, so the link will
1124  * @state: ASPM link state to disable
1136 	struct pcie_link_state *link;  in pcie_aspm_set_policy()  local
1149 	list_for_each_entry(link, &link_list, sibling) {  in pcie_aspm_set_policy()
1150 		pcie_config_aspm_link(link, policy_to_aspm_state(link));  in pcie_aspm_set_policy()
1151 		pcie_set_clkpm(link, policy_to_clkpm_state(link));  in pcie_aspm_set_policy()
1209 	struct pcie_link_state *link, *root = pdev->link_state->root;  in link_state_store()  local
1222 	list_for_each_entry(link, &link_list, sibling) {  in link_state_store()
1223 		if (link->root != root)  in link_state_store()
1225 		pcie_config_aspm_link(link, state);  in link_state_store()