2  * cpuidle.c - core cpuidle infrastructure
4  * (C) 2006-2007 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>
11 #include "linux/percpu-defs.h"
24 #include <linux/suspend.h>
25 #include <linux/tick.h>
54 	return off || !initialized || !drv || !dev || !dev->enabled;  in cpuidle_not_available()
58  * cpuidle_play_dead - cpu off-lining
60  * Returns in case of an error or no driver
69 		return -ENODEV;  in cpuidle_play_dead()
71 	/* Find lowest-power state that supports long-term idle */  in cpuidle_play_dead()
72 	for (i = drv->state_count - 1; i >= 0; i--)  in cpuidle_play_dead()
73 		if (drv->states[i].enter_dead)  in cpuidle_play_dead()
74 			return drv->states[i].enter_dead(dev, i);  in cpuidle_play_dead()
76 	return -ENODEV;  in cpuidle_play_dead()
88 	for (i = 1; i < drv->state_count; i++) {  in find_deepest_state()
89 		struct cpuidle_state *s = &drv->states[i];  in find_deepest_state()
91 		if (dev->states_usage[i].disable ||  in find_deepest_state()
92 		    s->exit_latency_ns <= latency_req ||  in find_deepest_state()
93 		    s->exit_latency_ns > max_latency_ns ||  in find_deepest_state()
94 		    (s->flags & forbidden_flags) ||  in find_deepest_state()
95 		    (s2idle && !s->enter_s2idle))  in find_deepest_state()
98 		latency_req = s->exit_latency_ns;  in find_deepest_state()
105  * cpuidle_use_deepest_state - Set/unset governor override mode.
106  * @latency_limit_ns: Idle state exit latency limit (or no override if 0).
119 		dev->forced_idle_latency_limit_ns = latency_limit_ns;  in cpuidle_use_deepest_state()
124  * cpuidle_find_deepest_state - Find the deepest available idle state.
143 	struct cpuidle_state *target_state = &drv->states[index];  in enter_s2idle_proper()
154 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in enter_s2idle_proper()
156 	target_state->enter_s2idle(dev, drv, index);  in enter_s2idle_proper()
159 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in enter_s2idle_proper()
166 	dev->states_usage[index].s2idle_time += ktime_us_delta(time_end, time_start);  in enter_s2idle_proper()
167 	dev->states_usage[index].s2idle_usage++;  in enter_s2idle_proper()
171  * cpuidle_enter_s2idle - Enter an idle state suitable for suspend-to-idle.
175  * If there are states with the ->enter_s2idle callback, find the deepest of
176  * them and enter it with frozen tick.
183 	 * Find the deepest state with ->enter_s2idle present, which guarantees  in cpuidle_enter_s2idle()
184 	 * that interrupts won't be enabled when it exits and allows the tick to  in cpuidle_enter_s2idle()
197  * cpuidle_enter_state - enter the state and update stats
200  * @index: index into the states table in @drv of the state to enter
207 	struct cpuidle_state *target_state = &drv->states[index];  in cpuidle_enter_state()
208 	bool broadcast = !!(target_state->flags & CPUIDLE_FLAG_TIMER_STOP);  in cpuidle_enter_state()
217 		index = find_deepest_state(drv, dev, target_state->exit_latency_ns,  in cpuidle_enter_state()
221 			return -EBUSY;  in cpuidle_enter_state()
223 		target_state = &drv->states[index];  in cpuidle_enter_state()
227 	if (target_state->flags & CPUIDLE_FLAG_TLB_FLUSHED)  in cpuidle_enter_state()
228 		leave_mm(dev->cpu);  in cpuidle_enter_state()
233 	trace_cpu_idle(index, dev->cpu);  in cpuidle_enter_state()
237 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in cpuidle_enter_state()
239 	entered_state = target_state->enter(dev, drv, index);  in cpuidle_enter_state()
240 	if (!(target_state->flags & CPUIDLE_FLAG_RCU_IDLE))  in cpuidle_enter_state()
246 	trace_cpu_idle(PWR_EVENT_EXIT, dev->cpu);  in cpuidle_enter_state()
248 	/* The cpu is no longer idle or about to enter idle. */  in cpuidle_enter_state()
262 		s64 diff, delay = drv->states[entered_state].exit_latency_ns;  in cpuidle_enter_state()
268 		 * but that results in multiple copies of same code.  in cpuidle_enter_state()
272 		dev->last_residency_ns = diff;  in cpuidle_enter_state()
273 		dev->states_usage[entered_state].time_ns += diff;  in cpuidle_enter_state()
274 		dev->states_usage[entered_state].usage++;  in cpuidle_enter_state()
276 		if (diff < drv->states[entered_state].target_residency_ns) {  in cpuidle_enter_state()
277 			for (i = entered_state - 1; i >= 0; i--) {  in cpuidle_enter_state()
278 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
282 				dev->states_usage[entered_state].above++;  in cpuidle_enter_state()
283 				trace_cpu_idle_miss(dev->cpu, entered_state, false);  in cpuidle_enter_state()
287 			for (i = entered_state + 1; i < drv->state_count; i++) {  in cpuidle_enter_state()
288 				if (dev->states_usage[i].disable)  in cpuidle_enter_state()
295 				if (diff - delay >= drv->states[i].target_residency_ns) {  in cpuidle_enter_state()
296 					dev->states_usage[entered_state].below++;  in cpuidle_enter_state()
297 					trace_cpu_idle_miss(dev->cpu, entered_state, true);  in cpuidle_enter_state()
304 		dev->last_residency_ns = 0;  in cpuidle_enter_state()
305 		dev->states_usage[index].rejected++;  in cpuidle_enter_state()
312  * cpuidle_select - ask the cpuidle framework to choose an idle state
316  * @stop_tick: indication on whether or not to stop the tick
321  * 'false' boolean value if the scheduler tick should not be stopped before
327 	return cpuidle_curr_governor->select(drv, dev, stop_tick);  in cpuidle_select()
331  * cpuidle_enter - enter into the specified idle state
335  * @index: the index in the idle state table
337  * Returns the index in the idle state, < 0 in case of error.
346 	 * Store the next hrtimer, which becomes either next tick or the next  in cpuidle_enter()
349 	 * ->select() callback have decided, whether to stop the tick or not.  in cpuidle_enter()
351 	WRITE_ONCE(dev->next_hrtimer, tick_nohz_get_next_hrtimer());  in cpuidle_enter()
358 	WRITE_ONCE(dev->next_hrtimer, 0);  in cpuidle_enter()
363  * cpuidle_reflect - tell the underlying governor what was the state
364  * we were in
367  * @index: the index in the idle state table
372 	if (cpuidle_curr_governor->reflect && index >= 0)  in cpuidle_reflect()
373 		cpuidle_curr_governor->reflect(dev, index);  in cpuidle_reflect()
378  * for most users, the time for a single ping-pong workload like
382  * perf bench sched pipe -l 10000
390  * cpuidle_poll_time - return amount of time to poll for,
391  * governors can override dev->poll_limit_ns if necessary
405 	if (dev->poll_limit_ns)  in cpuidle_poll_time()
406 		return dev->poll_limit_ns;  in cpuidle_poll_time()
409 	for (i = 1; i < drv->state_count; i++) {  in cpuidle_poll_time()
412 		if (dev->states_usage[i].disable)  in cpuidle_poll_time()
415 		state_limit = drv->states[i].target_residency_ns;  in cpuidle_poll_time()
423 	dev->poll_limit_ns = limit_ns;  in cpuidle_poll_time()
425 	return dev->poll_limit_ns;  in cpuidle_poll_time()
429  * cpuidle_install_idle_handler - installs the cpuidle idle loop handler
441  * cpuidle_uninstall_idle_handler - uninstalls the cpuidle idle loop handler
458  * cpuidle_pause_and_lock - temporarily disables CPUIDLE
469  * cpuidle_resume_and_unlock - resumes CPUIDLE operation
479 /* Currently used in suspend/resume path to suspend cpuidle */
487 /* Currently used in suspend/resume path to resume cpuidle */
496  * cpuidle_enable_device - enables idle PM for a CPU
508 		return -EINVAL;  in cpuidle_enable_device()
510 	if (dev->enabled)  in cpuidle_enable_device()
514 		return -EIO;  in cpuidle_enable_device()
519 		return -EIO;  in cpuidle_enable_device()
521 	if (!dev->registered)  in cpuidle_enable_device()
522 		return -EINVAL;  in cpuidle_enable_device()
528 	if (cpuidle_curr_governor->enable) {  in cpuidle_enable_device()
529 		ret = cpuidle_curr_governor->enable(drv, dev);  in cpuidle_enable_device()
536 	dev->enabled = 1;  in cpuidle_enable_device()
550  * cpuidle_disable_device - disables idle PM for a CPU
560 	if (!dev || !dev->enabled)  in cpuidle_disable_device()
566 	dev->enabled = 0;  in cpuidle_disable_device()
568 	if (cpuidle_curr_governor->disable)  in cpuidle_disable_device()
569 		cpuidle_curr_governor->disable(drv, dev);  in cpuidle_disable_device()
572 	enabled_devices--;  in cpuidle_disable_device()
581 	list_del(&dev->device_list);  in __cpuidle_unregister_device()
582 	per_cpu(cpuidle_devices, dev->cpu) = NULL;  in __cpuidle_unregister_device()
583 	module_put(drv->owner);  in __cpuidle_unregister_device()
585 	dev->registered = 0;  in __cpuidle_unregister_device()
590 	memset(dev->states_usage, 0, sizeof(dev->states_usage));  in __cpuidle_device_init()
591 	dev->last_residency_ns = 0;  in __cpuidle_device_init()
592 	dev->next_hrtimer = 0;  in __cpuidle_device_init()
596  * __cpuidle_register_device - internal register function called before register
607 	if (!try_module_get(drv->owner))  in __cpuidle_register_device()
608 		return -EINVAL;  in __cpuidle_register_device()
610 	for (i = 0; i < drv->state_count; i++) {  in __cpuidle_register_device()
611 		if (drv->states[i].flags & CPUIDLE_FLAG_UNUSABLE)  in __cpuidle_register_device()
612 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_DRIVER;  in __cpuidle_register_device()
614 		if (drv->states[i].flags & CPUIDLE_FLAG_OFF)  in __cpuidle_register_device()
615 			dev->states_usage[i].disable |= CPUIDLE_STATE_DISABLED_BY_USER;  in __cpuidle_register_device()
618 	per_cpu(cpuidle_devices, dev->cpu) = dev;  in __cpuidle_register_device()
619 	list_add(&dev->device_list, &cpuidle_detected_devices);  in __cpuidle_register_device()
625 		dev->registered = 1;  in __cpuidle_register_device()
631  * cpuidle_register_device - registers a CPU's idle PM feature
636 	int ret = -EBUSY;  in cpuidle_register_device()
639 		return -EINVAL;  in cpuidle_register_device()
643 	if (dev->registered)  in cpuidle_register_device()
677  * cpuidle_unregister_device - unregisters a CPU's idle PM feature
682 	if (!dev || dev->registered == 0)  in cpuidle_unregister_device()
712 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_unregister()
724  * initialization pattern there are in the arch specific drivers. The
725  * devices is globally defined in this file.
744 	for_each_cpu(cpu, drv->cpumask) {  in cpuidle_register()
746 		device->cpu = cpu;  in cpuidle_register()
751 		 * enabled in the kernel even if the cpuidle driver does not  in cpuidle_register()
755 			device->coupled_cpus = *coupled_cpus;  in cpuidle_register()
772  * cpuidle_init - core initializer
777 		return -ENODEV;  in cpuidle_init()