1 // SPDX-License-Identifier: GPL-2.0-only
3  * intel_idle.c - native hardware idle loop for modern Intel processors
5  * Copyright (c) 2013 - 2020, Intel Corporation.
12  * in lieu of the legacy ACPI processor_idle driver.  The intent is to
23  *	for preventing entry into deep C-states
25  * CPU will flush caches as needed when entering a C-state via MWAIT
26  *	(in contrast to entering ACPI C3, in which case the WBINVD
33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
35  * Have not seen issues with suspend, but may need same workaround here.
39 /* un-comment DEBUG to enable pr_debug() statements */
47 #include <linux/tick.h>
55 #include <asm/intel-family.h>
56 #include <asm/nospec-branch.h>
68 static int max_cstate = CPUIDLE_STATE_MAX - 1;
86 	 * Hardware C-state auto-demotion may not always be optimal.
101  * Enable interrupts before entering the C-state. On some platforms and for
102  * some C-states, this may measurably decrease interrupt latency.
118  * Initialize large xstate for the C6-state entrance.
123  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
124  * the C-state (top nibble) and sub-state (bottom nibble)
135 	struct cpuidle_state *state = &drv->states[index];  in __intel_idle()
136 	unsigned long eax = flg2MWAIT(state->flags);  in __intel_idle()
145  * intel_idle - Ask the processor to enter the given idle state.
153  * If the local APIC timer is not known to be reliable in the target idle state,
154  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
174 	 * tick interrupt followed by __do_softirq(). Use local_irq_disable()  in intel_idle_irq()
208  * intel_idle_s2idle - Ask the processor to enter the given idle state.
216  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
217  * scheduler tick and suspended scheduler clock on the target CPU.
223 	struct cpuidle_state *state = &drv->states[index];  in intel_idle_s2idle()
224 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle_s2idle()
226 	if (state->flags & CPUIDLE_FLAG_INIT_XSTATE)  in intel_idle_s2idle()
833  * C1E is enabled only if "C1E promotion" bit is set in MSR_IA32_POWER_CTL.
834  * But in this case there is effectively no C1, because C1 requests are
836  * and C1E requests end up with C1, so there is effectively no C1E.
1219  * C6, and this is indicated in the CPUID mwait leaf.
1458 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle_state_needs_timer_stop()
1464 	 * Switch over to one-shot tick broadcast if the target C-state  in intel_idle_state_needs_timer_stop()
1477 static bool force_use_acpi __read_mostly; /* No effect if no_acpi is set. */
1484  * intel_idle_cst_usable - Check if the _CST information can be used.
1486  * Check if all of the C-states listed by _CST in the max_cstate range are
1499 		if (cx->entry_method != ACPI_CSTATE_FFH)  in intel_idle_cst_usable()
1521 		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))  in intel_idle_acpi_cst_extract()
1552 		if (intel_idle_max_cstate_reached(cstate - 1))  in intel_idle_init_cstates_acpi()
1557 		state = &drv->states[drv->state_count++];  in intel_idle_init_cstates_acpi()
1559 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);  in intel_idle_init_cstates_acpi()
1560 		strscpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);  in intel_idle_init_cstates_acpi()
1561 		state->exit_latency = cx->latency;  in intel_idle_init_cstates_acpi()
1563 		 * For C1-type C-states use the same number for both the exit  in intel_idle_init_cstates_acpi()
1565 		 * C1 in the majority of the static C-states tables above.  in intel_idle_init_cstates_acpi()
1566 		 * For the other types of C-states, however, set the target  in intel_idle_init_cstates_acpi()
1568 		 * a reasonable balance between energy-efficiency and  in intel_idle_init_cstates_acpi()
1569 		 * performance in the majority of interesting cases.  in intel_idle_init_cstates_acpi()
1571 		state->target_residency = cx->latency;  in intel_idle_init_cstates_acpi()
1572 		if (cx->type > ACPI_STATE_C1)  in intel_idle_init_cstates_acpi()
1573 			state->target_residency *= 3;  in intel_idle_init_cstates_acpi()
1575 		state->flags = MWAIT2flg(cx->address);  in intel_idle_init_cstates_acpi()
1576 		if (cx->type > ACPI_STATE_C2)  in intel_idle_init_cstates_acpi()
1577 			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;  in intel_idle_init_cstates_acpi()
1580 			state->flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_acpi()
1583 			state->flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_acpi()
1585 		state->enter = intel_idle;  in intel_idle_init_cstates_acpi()
1586 		state->enter_s2idle = intel_idle_s2idle;  in intel_idle_init_cstates_acpi()
1595 	 * If there are no _CST C-states, do not disable any C-states by  in intel_idle_off_by_default()
1621  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1623  * Tune IVT multi-socket targets.
1628 	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */  in ivt_idle_state_table_update()
1650  * irtl_2_usec - IRTL to microseconds conversion.
1671  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1719  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1721  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1733 	/* if PC10 not present in CPUID.MWAIT.EDX */  in sklh_idle_state_table_update()
1739 	/* PC10 is not enabled in PKG C-state limit */  in sklh_idle_state_table_update()
1756 	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */  in sklh_idle_state_table_update()
1757 	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */  in sklh_idle_state_table_update()
1761  * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1771 	 * 000b: C0/C1 (no package C-state support)  in skx_idle_state_table_update()
1773 	 * 010b: C6 (non-retention)  in skx_idle_state_table_update()
1775 	 * 111b: No Package C state limits.  in skx_idle_state_table_update()
1781 		 * is disabled in BIOS. This is consistent  in skx_idle_state_table_update()
1791  * adl_idle_state_table_update - Adjust AlderLake idle states table.
1810  * spr_idle_state_table_update - Adjust Sapphire Rapids idle states table.
1817 	 * By default, the C6 state assumes the worst-case scenario of package  in spr_idle_state_table_update()
1836 	/* Ignore the C-state if there are NO sub-states in CPUID for it. */  in intel_idle_verify_cstate()
1841 		mark_tsc_unstable("TSC halts in idle states deeper than C2");  in intel_idle_verify_cstate()
1896 		drv->states[drv->state_count] = cpuidle_state_table[cstate];  in intel_idle_init_cstates_icpu()
1899 			drv->states[drv->state_count].enter = intel_idle_irq;  in intel_idle_init_cstates_icpu()
1904 			drv->states[drv->state_count].enter = intel_idle_ibrs;  in intel_idle_init_cstates_icpu()
1908 			drv->states[drv->state_count].enter = intel_idle_xstate;  in intel_idle_init_cstates_icpu()
1910 		if ((disabled_states_mask & BIT(drv->state_count)) ||  in intel_idle_init_cstates_icpu()
1911 		    ((icpu->use_acpi || force_use_acpi) &&  in intel_idle_init_cstates_icpu()
1914 			drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_icpu()
1916 		if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count]))  in intel_idle_init_cstates_icpu()
1917 			drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_icpu()
1919 		drv->state_count++;  in intel_idle_init_cstates_icpu()
1922 	if (icpu->byt_auto_demotion_disable_flag) {  in intel_idle_init_cstates_icpu()
1929  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
1937 		drv->states[0].flags |= CPUIDLE_FLAG_OFF;  in intel_idle_cpuidle_driver_init()
1939 	drv->state_count = 1;  in intel_idle_cpuidle_driver_init()
1975  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
1978  * Register a cpuidle device object for @cpu and update its MSRs in accordance
1986 	dev->cpu = cpu;  in intel_idle_cpu_init()
1990 		return -EIO;  in intel_idle_cpu_init()
2014 	 * driver in this case  in intel_idle_cpu_online()
2017 	if (!dev->registered)  in intel_idle_cpu_online()
2024  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
2042 		return -ENODEV;  in intel_idle_init()
2046 		return -EPERM;  in intel_idle_init()
2052 			pr_debug("Please enable MWAIT in BIOS SETUP\n");  in intel_idle_init()
2053 			return -ENODEV;  in intel_idle_init()
2058 			return -ENODEV;  in intel_idle_init()
2062 		return -ENODEV;  in intel_idle_init()
2069 			return -ENODEV;  in intel_idle_init()
2073 	icpu = (const struct idle_cpu *)id->driver_data;  in intel_idle_init()
2075 		cpuidle_state_table = icpu->state_table;  in intel_idle_init()
2076 		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;  in intel_idle_init()
2077 		if (icpu->disable_promotion_to_c1e)  in intel_idle_init()
2079 		if (icpu->use_acpi || force_use_acpi)  in intel_idle_init()
2082 		return -ENODEV;  in intel_idle_init()
2090 		return -ENOMEM;  in intel_idle_init()
2098 		       drv ? drv->name : "none");  in intel_idle_init()
2107 	pr_debug("Local APIC timer is reliable in %s\n",  in intel_idle_init()
2108 		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");  in intel_idle_init()
2124  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
2125  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
2130  * The positions of the bits that are set in this number are the indices of the
2132  * corresponding idle state directories in sysfs, "state0", "state1" ...
2138  * Some platforms come with mutually exclusive C-states, so that if one is
2139  * enabled, the other C-states must not be used. Example: C1 and C1E on
2141  * preferred C-states among the groups of mutually exclusive C-states - the
2142  * selected C-states will be registered, the other C-states from the mutually
2143  * exclusive group won't be registered. If the platform has no mutually
2144  * exclusive C-states, this parameter has no effect.