1 // SPDX-License-Identifier: GPL-2.0-only
3  * intel_idle.c - native hardware idle loop for modern Intel processors
5  * Copyright (c) 2013 - 2020, Intel Corporation.
20  * All CPUs have same idle states as boot CPU
23  *	for preventing entry into deep C-states
25  * CPU will flush caches as needed when entering a C-state via MWAIT
33  * ACPI has a .suspend hack to turn off deep c-statees during suspend
39 /* un-comment DEBUG to enable pr_debug() statements */
54 #include <asm/intel-family.h>
61 	.name = "intel_idle",
65 static int max_cstate = CPUIDLE_STATE_MAX - 1;
77 	 * Hardware C-state auto-demotion may not always be optimal.
92  * Enable this state by default even if the ACPI _CST does not list it.
97  * MWAIT takes an 8-bit "hint" in EAX "suggesting"
98  * the C-state (top nibble) and sub-state (bottom nibble)
101  * We store the hint at the top of our "flags" for each state.
107  * intel_idle - Ask the processor to enter the given idle state.
110  * @index: Target idle state index.
113  * @dev is idle and it can try to enter the idle state corresponding to @index.
115  * If the local APIC timer is not known to be reliable in the target idle state,
116  * enable one-shot tick broadcasting for the target CPU before executing MWAIT.
126 	struct cpuidle_state *state = &drv->states[index];  in intel_idle()  local
127 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle()
136  * intel_idle_s2idle - Ask the processor to enter the given idle state.
139  * @index: Target idle state index.
142  * @dev is idle and it can try to enter the idle state corresponding to @index.
144  * Invoked as a suspend-to-idle callback routine with frozen user space, frozen
150 	unsigned long eax = flg2MWAIT(drv->states[index].flags);  in intel_idle_s2idle()
165 		.name = "C1",
173 		.name = "C1E",
181 		.name = "C3",
189 		.name = "C6",
202 		.name = "C1",
210 		.name = "C1E",
218 		.name = "C3",
226 		.name = "C6",
234 		.name = "C7",
247 		.name = "C1",
255 		.name = "C6N",
263 		.name = "C6S",
271 		.name = "C7",
279 		.name = "C7S",
292 		.name = "C1",
300 		.name = "C6N",
308 		.name = "C6S",
316 		.name = "C7",
324 		.name = "C7S",
337 		.name = "C1",
345 		.name = "C1E",
353 		.name = "C3",
361 		.name = "C6",
369 		.name = "C7",
382 		.name = "C1",
390 		.name = "C1E",
398 		.name = "C3",
406 		.name = "C6",
419 		.name = "C1",
427 		.name = "C1E",
435 		.name = "C3",
443 		.name = "C6",
456 		.name = "C1",
464 		.name = "C1E",
472 		.name = "C3",
480 		.name = "C6",
493 		.name = "C1",
501 		.name = "C1E",
509 		.name = "C3",
517 		.name = "C6",
525 		.name = "C7s",
533 		.name = "C8",
541 		.name = "C9",
549 		.name = "C10",
561 		.name = "C1",
569 		.name = "C1E",
577 		.name = "C3",
585 		.name = "C6",
593 		.name = "C7s",
601 		.name = "C8",
609 		.name = "C9",
617 		.name = "C10",
630 		.name = "C1",
638 		.name = "C1E",
646 		.name = "C3",
654 		.name = "C6",
662 		.name = "C7s",
670 		.name = "C8",
678 		.name = "C9",
686 		.name = "C10",
699 		.name = "C1",
707 		.name = "C1E",
715 		.name = "C6",
728 		.name = "C1",
736 		.name = "C1E",
744 		.name = "C6",
757 		.name = "C1E",
765 		.name = "C2",
773 		.name = "C4",
781 		.name = "C6",
793 		.name = "C1",
801 		.name = "C4",
809 		.name = "C6",
817 		.name = "C7",
825 		.name = "C9",
837 		.name = "C1",
845 		.name = "C6",
857 		.name = "C1",
865 		.name = "C6",
878 		.name = "C1",
886 		.name = "C1E",
894 		.name = "C6",
902 		.name = "C7s",
910 		.name = "C8",
918 		.name = "C9",
926 		.name = "C10",
939 		.name = "C1",
947 		.name = "C1E",
955 		.name = "C6",
972 		.name = "C1",
980 		.name = "C1E",
988 		.name = "C6",
1183 static bool __init intel_idle_state_needs_timer_stop(struct cpuidle_state *state)  in intel_idle_state_needs_timer_stop()  argument
1185 	unsigned long eax = flg2MWAIT(state->flags);  in intel_idle_state_needs_timer_stop()
1191 	 * Switch over to one-shot tick broadcast if the target C-state  in intel_idle_state_needs_timer_stop()
1202 MODULE_PARM_DESC(no_acpi, "Do not use ACPI _CST for building the idle states list");
1206 MODULE_PARM_DESC(use_acpi, "Use ACPI _CST for building the idle states list");
1211  * intel_idle_cst_usable - Check if the _CST information can be used.
1213  * Check if all of the C-states listed by _CST in the max_cstate range are
1226 		if (cx->entry_method != ACPI_CSTATE_FFH)  in intel_idle_cst_usable()
1248 		if (acpi_processor_evaluate_cst(pr->handle, cpu, &acpi_state_table))  in intel_idle_acpi_cst_extract()
1277 		struct cpuidle_state *state;  in intel_idle_init_cstates_acpi()  local
1279 		if (intel_idle_max_cstate_reached(cstate - 1))  in intel_idle_init_cstates_acpi()
1284 		state = &drv->states[drv->state_count++];  in intel_idle_init_cstates_acpi()
1286 		snprintf(state->name, CPUIDLE_NAME_LEN, "C%d_ACPI", cstate);  in intel_idle_init_cstates_acpi()
1287 		strlcpy(state->desc, cx->desc, CPUIDLE_DESC_LEN);  in intel_idle_init_cstates_acpi()
1288 		state->exit_latency = cx->latency;  in intel_idle_init_cstates_acpi()
1290 		 * For C1-type C-states use the same number for both the exit  in intel_idle_init_cstates_acpi()
1292 		 * C1 in the majority of the static C-states tables above.  in intel_idle_init_cstates_acpi()
1293 		 * For the other types of C-states, however, set the target  in intel_idle_init_cstates_acpi()
1295 		 * a reasonable balance between energy-efficiency and  in intel_idle_init_cstates_acpi()
1298 		state->target_residency = cx->latency;  in intel_idle_init_cstates_acpi()
1299 		if (cx->type > ACPI_STATE_C1)  in intel_idle_init_cstates_acpi()
1300 			state->target_residency *= 3;  in intel_idle_init_cstates_acpi()
1302 		state->flags = MWAIT2flg(cx->address);  in intel_idle_init_cstates_acpi()
1303 		if (cx->type > ACPI_STATE_C2)  in intel_idle_init_cstates_acpi()
1304 			state->flags |= CPUIDLE_FLAG_TLB_FLUSHED;  in intel_idle_init_cstates_acpi()
1307 			state->flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_acpi()
1309 		if (intel_idle_state_needs_timer_stop(state))  in intel_idle_init_cstates_acpi()
1310 			state->flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_acpi()
1312 		state->enter = intel_idle;  in intel_idle_init_cstates_acpi()
1313 		state->enter_s2idle = intel_idle_s2idle;  in intel_idle_init_cstates_acpi()
1322 	 * If there are no _CST C-states, do not disable any C-states by  in intel_idle_off_by_default()
1348  * ivt_idle_state_table_update - Tune the idle states table for Ivy Town.
1350  * Tune IVT multi-socket targets.
1355 	/* IVT uses a different table for 1-2, 3-4, and > 4 sockets */  in ivt_idle_state_table_update()
1377  * irtl_2_usec - IRTL to microseconds conversion.
1398  * bxt_idle_state_table_update - Fix up the Broxton idle states table.
1446  * sklh_idle_state_table_update - Fix up the Sky Lake idle states table.
1448  * On SKL-H (model 0x5e) skip C8 and C9 if C10 is enabled and SGX disabled.
1466 	/* PC10 is not enabled in PKG C-state limit */  in sklh_idle_state_table_update()
1483 	skl_cstates[5].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C8-SKL */  in sklh_idle_state_table_update()
1484 	skl_cstates[6].flags |= CPUIDLE_FLAG_UNUSABLE;	/* C9-SKL */  in sklh_idle_state_table_update()
1488  * skx_idle_state_table_update - Adjust the Sky Lake/Cascade Lake
1489  * idle states table.
1498 	 * 000b: C0/C1 (no package C-state support)  in skx_idle_state_table_update()
1500 	 * 010b: C6 (non-retention)  in skx_idle_state_table_update()
1502 	 * 111b: No Package C state limits.  in skx_idle_state_table_update()
1523 	/* Ignore the C-state if there are NO sub-states in CPUID for it. */  in intel_idle_verify_cstate()
1528 		mark_tsc_unstable("TSC halts in idle states deeper than C2");  in intel_idle_verify_cstate()
1563 		/* If marked as unusable, skip this state. */  in intel_idle_init_cstates_icpu()
1565 			pr_debug("state %s is disabled\n",  in intel_idle_init_cstates_icpu()
1566 				 cpuidle_state_table[cstate].name);  in intel_idle_init_cstates_icpu()
1575 		drv->states[drv->state_count] = cpuidle_state_table[cstate];  in intel_idle_init_cstates_icpu()
1577 		if ((disabled_states_mask & BIT(drv->state_count)) ||  in intel_idle_init_cstates_icpu()
1578 		    ((icpu->use_acpi || force_use_acpi) &&  in intel_idle_init_cstates_icpu()
1581 			drv->states[drv->state_count].flags |= CPUIDLE_FLAG_OFF;  in intel_idle_init_cstates_icpu()
1583 		if (intel_idle_state_needs_timer_stop(&drv->states[drv->state_count]))  in intel_idle_init_cstates_icpu()
1584 			drv->states[drv->state_count].flags |= CPUIDLE_FLAG_TIMER_STOP;  in intel_idle_init_cstates_icpu()
1586 		drv->state_count++;  in intel_idle_init_cstates_icpu()
1589 	if (icpu->byt_auto_demotion_disable_flag) {  in intel_idle_init_cstates_icpu()
1596  * intel_idle_cpuidle_driver_init - Create the list of available idle states.
1604 		drv->states[0].flags |= CPUIDLE_FLAG_OFF;  in intel_idle_cpuidle_driver_init()
1606 	drv->state_count = 1;  in intel_idle_cpuidle_driver_init()
1633  * intel_idle_cpu_init - Register the target CPU with the cpuidle core.
1644 	dev->cpu = cpu;  in intel_idle_cpu_init()
1648 		return -EIO;  in intel_idle_cpu_init()
1673 	if (!dev->registered)  in intel_idle_cpu_online()
1680  * intel_idle_cpuidle_devices_uninit - Unregister all cpuidle devices.
1696 	/* Do not load intel_idle at all for now if idle= is passed */  in intel_idle_init()
1698 		return -ENODEV;  in intel_idle_init()
1702 		return -EPERM;  in intel_idle_init()
1709 			return -ENODEV;  in intel_idle_init()
1714 			return -ENODEV;  in intel_idle_init()
1718 		return -ENODEV;  in intel_idle_init()
1725 			return -ENODEV;  in intel_idle_init()
1729 	icpu = (const struct idle_cpu *)id->driver_data;  in intel_idle_init()
1731 		cpuidle_state_table = icpu->state_table;  in intel_idle_init()
1732 		auto_demotion_disable_flags = icpu->auto_demotion_disable_flags;  in intel_idle_init()
1733 		disable_promotion_to_c1e = icpu->disable_promotion_to_c1e;  in intel_idle_init()
1734 		if (icpu->use_acpi || force_use_acpi)  in intel_idle_init()
1737 		return -ENODEV;  in intel_idle_init()
1745 		return -ENOMEM;  in intel_idle_init()
1753 		       drv ? drv->name : "none");  in intel_idle_init()
1757 	retval = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "idle/intel:online",  in intel_idle_init()
1763 		 boot_cpu_has(X86_FEATURE_ARAT) ? "all C-states" : "C1");  in intel_idle_init()
1779  * support "intel_idle.max_cstate=..." at boot and also a read-only export of
1780  * it at /sys/module/intel_idle/parameters/max_cstate -- so using module_param
1786  * idle states to be disabled by default (as reflected by the names of the
1787  * corresponding idle state directories in sysfs, "state0", "state1" ...
1788  * "state<i>" ..., where <i> is the index of the given state).
1791 MODULE_PARM_DESC(states_off, "Mask of disabled idle states");