Lines Matching +full:cpu +full:- +full:idle +full:- +full:states
1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * coupled.c - helper functions to enter the same idle state on multiple cpus
11 #include <linux/cpu.h>
21 * DOC: Coupled cpuidle states
25 * sequencing restrictions (on Tegra 2, cpu 0 must be the last to
26 * power down), or due to HW bugs (on OMAP4460, a cpu powering up
27 * will corrupt the gic state unless the other cpu runs a work
28 * around). Each cpu has a power state that it can enter without
29 * coordinating with the other cpu (usually Wait For Interrupt, or
30 * WFI), and one or more "coupled" power states that affect blocks
35 * This file implements a solution, where each cpu will wait in the
40 * Once all cpus are ready to enter idle, they are woken by an smp
42 * cpus will find work to do, and choose not to enter idle. A
45 * each cpu will increment the ready counter, and continue once the
47 * cpu exits idle, the other cpus will decrement their counter and
50 * requested_state stores the deepest coupled idle state each cpu
51 * is ready for. It is assumed that the states are indexed from
54 * variable is not locked. It is only written from the cpu that
55 * it stores (or by the on/offlining cpu if that cpu is offline),
56 * and only read after all the cpus are ready for the coupled idle
62 * the waiting loop, in the ready loop, or in the coupled idle state.
64 * or in the coupled idle state.
66 * To use coupled cpuidle states, a cpuidle driver must:
71 * set in the struct cpuidle_device for each cpu.
82 * should ensure that the cpus all abort together if any cpu tries
88 * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
90 * @requested_state: array of requested states for cpus in the coupled set
95 * @prevent: flag to prevent coupled idle while a cpu is hotplugging
109 #define WAITING_MASK (MAX_WAITING_CPUS - 1)
112 #define CPUIDLE_COUPLED_NOT_IDLE (-1)
118 * __smp_call_function_single with the per cpu call_single_data_t struct already
125 * The cpuidle_coupled_poked mask is used to ensure that each cpu has been poked
132 * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
133 * @dev: cpuidle_device of the calling cpu
141 * The atomic variable must be initialized to 0 before any cpu calls
142 * this function, will be reset to 0 before any cpu returns from this function.
144 * Must only be called from within a coupled idle state handler
151 int n = dev->coupled->online_count; in cpuidle_coupled_parallel_barrier()
169 * cpuidle_state_is_coupled - check if a state is part of a coupled set
171 * @state: index of the target state in drv->states
177 return drv->states[state].flags & CPUIDLE_FLAG_COUPLED; in cpuidle_state_is_coupled()
181 * cpuidle_coupled_state_verify - check if the coupled states are correctly set.
185 * * -EINVAL if any coupled state(safe_state_index) is wrongly set.
191 for (i = drv->state_count - 1; i >= 0; i--) { in cpuidle_coupled_state_verify()
193 (drv->safe_state_index == i || in cpuidle_coupled_state_verify()
194 drv->safe_state_index < 0 || in cpuidle_coupled_state_verify()
195 drv->safe_state_index >= drv->state_count)) in cpuidle_coupled_state_verify()
196 return -EINVAL; in cpuidle_coupled_state_verify()
203 * cpuidle_coupled_set_ready - mark a cpu as ready
204 * @coupled: the struct coupled that contains the current cpu
208 atomic_add(MAX_WAITING_CPUS, &coupled->ready_waiting_counts); in cpuidle_coupled_set_ready()
212 * cpuidle_coupled_set_not_ready - mark a cpu as not ready
213 * @coupled: the struct coupled that contains the current cpu
216 * is equal to the number of online cpus. Prevents a race where one cpu
217 * decrements the waiting counter and then re-increments it just before another
218 * cpu has decremented its ready counter, leading to the ready counter going
219 * down from the number of online cpus without going through the coupled idle
222 * Returns 0 if the counter was decremented successfully, -EINVAL if the ready
231 all = coupled->online_count | (coupled->online_count << WAITING_BITS); in cpuidle_coupled_set_not_ready()
232 ret = atomic_add_unless(&coupled->ready_waiting_counts, in cpuidle_coupled_set_not_ready()
233 -MAX_WAITING_CPUS, all); in cpuidle_coupled_set_not_ready()
235 return ret ? 0 : -EINVAL; in cpuidle_coupled_set_not_ready()
239 * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
240 * @coupled: the struct coupled that contains the current cpu
246 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS; in cpuidle_coupled_no_cpus_ready()
251 * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
252 * @coupled: the struct coupled that contains the current cpu
258 int r = atomic_read(&coupled->ready_waiting_counts) >> WAITING_BITS; in cpuidle_coupled_cpus_ready()
259 return r == coupled->online_count; in cpuidle_coupled_cpus_ready()
263 * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
264 * @coupled: the struct coupled that contains the current cpu
270 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_cpus_waiting()
271 return w == coupled->online_count; in cpuidle_coupled_cpus_waiting()
275 * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
276 * @coupled: the struct coupled that contains the current cpu
282 int w = atomic_read(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_no_cpus_waiting()
287 * cpuidle_coupled_get_state - determine the deepest idle state
288 * @dev: struct cpuidle_device for this cpu
289 * @coupled: the struct coupled that contains the current cpu
291 * Returns the deepest idle state that all coupled cpus can enter
306 for_each_cpu(i, &coupled->coupled_cpus) in cpuidle_coupled_get_state()
307 if (cpu_online(i) && coupled->requested_state[i] < state) in cpuidle_coupled_get_state()
308 state = coupled->requested_state[i]; in cpuidle_coupled_get_state()
315 int cpu = (unsigned long)info; in cpuidle_coupled_handle_poke() local
316 cpumask_set_cpu(cpu, &cpuidle_coupled_poked); in cpuidle_coupled_handle_poke()
317 cpumask_clear_cpu(cpu, &cpuidle_coupled_poke_pending); in cpuidle_coupled_handle_poke()
321 * cpuidle_coupled_poke - wake up a cpu that may be waiting
322 * @cpu: target cpu
324 * Ensures that the target cpu exits it's waiting idle state (if it is in it)
325 * and will see updates to waiting_count before it re-enters it's waiting idle
328 * If cpuidle_coupled_poked_mask is already set for the target cpu, that cpu
329 * either has or will soon have a pending IPI that will wake it out of idle,
330 * or it is currently processing the IPI and is not in idle.
332 static void cpuidle_coupled_poke(int cpu) in cpuidle_coupled_poke() argument
334 call_single_data_t *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu); in cpuidle_coupled_poke()
336 if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending)) in cpuidle_coupled_poke()
337 smp_call_function_single_async(cpu, csd); in cpuidle_coupled_poke()
341 * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
342 * @this_cpu: target cpu
343 * @coupled: the struct coupled that contains the current cpu
350 int cpu; in cpuidle_coupled_poke_others() local
352 for_each_cpu(cpu, &coupled->coupled_cpus) in cpuidle_coupled_poke_others()
353 if (cpu != this_cpu && cpu_online(cpu)) in cpuidle_coupled_poke_others()
354 cpuidle_coupled_poke(cpu); in cpuidle_coupled_poke_others()
358 * cpuidle_coupled_set_waiting - mark this cpu as in the wait loop
359 * @cpu: target cpu
360 * @coupled: the struct coupled that contains the current cpu
361 * @next_state: the index in drv->states of the requested state for this cpu
363 * Updates the requested idle state for the specified cpuidle device.
366 static int cpuidle_coupled_set_waiting(int cpu, in cpuidle_coupled_set_waiting() argument
369 coupled->requested_state[cpu] = next_state; in cpuidle_coupled_set_waiting()
375 return atomic_inc_return(&coupled->ready_waiting_counts) & WAITING_MASK; in cpuidle_coupled_set_waiting()
379 * cpuidle_coupled_set_not_waiting - mark this cpu as leaving the wait loop
380 * @cpu: target cpu
381 * @coupled: the struct coupled that contains the current cpu
383 * Removes the requested idle state for the specified cpuidle device.
385 static void cpuidle_coupled_set_not_waiting(int cpu, in cpuidle_coupled_set_not_waiting() argument
392 * notice that this cpu has cleared it's requested_state. in cpuidle_coupled_set_not_waiting()
394 atomic_dec(&coupled->ready_waiting_counts); in cpuidle_coupled_set_not_waiting()
396 coupled->requested_state[cpu] = CPUIDLE_COUPLED_NOT_IDLE; in cpuidle_coupled_set_not_waiting()
400 * cpuidle_coupled_set_done - mark this cpu as leaving the ready loop
401 * @cpu: the current cpu
402 * @coupled: the struct coupled that contains the current cpu
404 * Marks this cpu as no longer in the ready and waiting loops. Decrements
405 * the waiting count first to prevent another cpu looping back in and seeing
406 * this cpu as waiting just before it exits idle.
408 static void cpuidle_coupled_set_done(int cpu, struct cpuidle_coupled *coupled) in cpuidle_coupled_set_done() argument
410 cpuidle_coupled_set_not_waiting(cpu, coupled); in cpuidle_coupled_set_done()
411 atomic_sub(MAX_WAITING_CPUS, &coupled->ready_waiting_counts); in cpuidle_coupled_set_done()
415 * cpuidle_coupled_clear_pokes - spin until the poke interrupt is processed
416 * @cpu: this cpu
423 * the interrupt didn't schedule work that should take the cpu out of idle.
427 static int cpuidle_coupled_clear_pokes(int cpu) in cpuidle_coupled_clear_pokes() argument
429 if (!cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending)) in cpuidle_coupled_clear_pokes()
433 while (cpumask_test_cpu(cpu, &cpuidle_coupled_poke_pending)) in cpuidle_coupled_clear_pokes()
445 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus); in cpuidle_coupled_any_pokes_pending()
452 * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
453 * @dev: struct cpuidle_device for the current cpu
455 * @next_state: index of the requested state in drv->states
462 * all the other cpus to call this function. Once all coupled cpus are idle,
463 * the second stage will start. Each coupled cpu will spin until all cpus have
467 * interrupts while preparing for idle, and it will always return with
473 int entered_state = -1; in cpuidle_enter_state_coupled()
474 struct cpuidle_coupled *coupled = dev->coupled; in cpuidle_enter_state_coupled()
478 return -EINVAL; in cpuidle_enter_state_coupled()
480 while (coupled->prevent) { in cpuidle_enter_state_coupled()
481 cpuidle_coupled_clear_pokes(dev->cpu); in cpuidle_enter_state_coupled()
487 drv->safe_state_index); in cpuidle_enter_state_coupled()
495 cpumask_clear_cpu(dev->cpu, &cpuidle_coupled_poked); in cpuidle_enter_state_coupled()
497 w = cpuidle_coupled_set_waiting(dev->cpu, coupled, next_state); in cpuidle_enter_state_coupled()
499 * If this is the last cpu to enter the waiting state, poke in cpuidle_enter_state_coupled()
505 if (w == coupled->online_count) { in cpuidle_enter_state_coupled()
506 cpumask_set_cpu(dev->cpu, &cpuidle_coupled_poked); in cpuidle_enter_state_coupled()
507 cpuidle_coupled_poke_others(dev->cpu, coupled); in cpuidle_enter_state_coupled()
512 * Wait for all coupled cpus to be idle, using the deepest state in cpuidle_enter_state_coupled()
513 * allowed for a single cpu. If this was not the poking cpu, wait in cpuidle_enter_state_coupled()
520 !cpumask_test_cpu(dev->cpu, &cpuidle_coupled_poked)) { in cpuidle_enter_state_coupled()
521 if (cpuidle_coupled_clear_pokes(dev->cpu)) in cpuidle_enter_state_coupled()
525 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
529 if (coupled->prevent) { in cpuidle_enter_state_coupled()
530 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
535 drv->safe_state_index); in cpuidle_enter_state_coupled()
539 cpuidle_coupled_clear_pokes(dev->cpu); in cpuidle_enter_state_coupled()
541 cpuidle_coupled_set_not_waiting(dev->cpu, coupled); in cpuidle_enter_state_coupled()
546 * Make sure final poke status for this cpu is visible before setting in cpuidle_enter_state_coupled()
547 * cpu as ready. in cpuidle_enter_state_coupled()
552 * All coupled cpus are probably idle. There is a small chance that in cpuidle_enter_state_coupled()
555 * cpu has incremented the ready counter, it cannot abort idle and must in cpuidle_enter_state_coupled()
557 * another cpu leaves idle and decrements the waiting counter. in cpuidle_enter_state_coupled()
562 /* Check if any other cpus bailed out of idle. */ in cpuidle_enter_state_coupled()
576 * There is a small chance that a cpu left and reentered idle after this in cpuidle_enter_state_coupled()
577 * cpu saw that all cpus were waiting. The cpu that reentered idle will in cpuidle_enter_state_coupled()
578 * have sent this cpu a poke, which will still be pending after the in cpuidle_enter_state_coupled()
580 * controller when entering the deep idle state. It's not possible to in cpuidle_enter_state_coupled()
583 * coupled idle state of all cpus and retry. in cpuidle_enter_state_coupled()
586 cpuidle_coupled_set_done(dev->cpu, coupled); in cpuidle_enter_state_coupled()
588 cpuidle_coupled_parallel_barrier(dev, &coupled->abort_barrier); in cpuidle_enter_state_coupled()
597 cpuidle_coupled_set_done(dev->cpu, coupled); in cpuidle_enter_state_coupled()
601 * Normal cpuidle states are expected to return with irqs enabled. in cpuidle_enter_state_coupled()
602 * That leads to an inefficiency where a cpu receiving an interrupt in cpuidle_enter_state_coupled()
603 * that brings it out of idle will process that interrupt before in cpuidle_enter_state_coupled()
604 * exiting the idle enter function and decrementing ready_count. All in cpuidle_enter_state_coupled()
605 * other cpus will need to spin waiting for the cpu that is processing in cpuidle_enter_state_coupled()
607 * all other cpus will loop back into the safe idle state instead of in cpuidle_enter_state_coupled()
610 * Calling local_irq_enable here allows coupled states to return with in cpuidle_enter_state_coupled()
617 * Wait until all coupled cpus have exited idle. There is no risk that in cpuidle_enter_state_coupled()
618 * a cpu exits and re-enters the ready state because this cpu has in cpuidle_enter_state_coupled()
630 cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus); in cpuidle_coupled_update_online_cpus()
631 coupled->online_count = cpumask_weight(&cpus); in cpuidle_coupled_update_online_cpus()
635 * cpuidle_coupled_register_device - register a coupled cpuidle device
636 * @dev: struct cpuidle_device for the current cpu
638 * Called from cpuidle_register_device to handle coupled idle init. Finds the
644 int cpu; in cpuidle_coupled_register_device() local
649 if (cpumask_empty(&dev->coupled_cpus)) in cpuidle_coupled_register_device()
652 for_each_cpu(cpu, &dev->coupled_cpus) { in cpuidle_coupled_register_device()
653 other_dev = per_cpu(cpuidle_devices, cpu); in cpuidle_coupled_register_device()
654 if (other_dev && other_dev->coupled) { in cpuidle_coupled_register_device()
655 coupled = other_dev->coupled; in cpuidle_coupled_register_device()
663 return -ENOMEM; in cpuidle_coupled_register_device()
665 coupled->coupled_cpus = dev->coupled_cpus; in cpuidle_coupled_register_device()
668 dev->coupled = coupled; in cpuidle_coupled_register_device()
669 if (WARN_ON(!cpumask_equal(&dev->coupled_cpus, &coupled->coupled_cpus))) in cpuidle_coupled_register_device()
670 coupled->prevent++; in cpuidle_coupled_register_device()
674 coupled->refcnt++; in cpuidle_coupled_register_device()
676 csd = &per_cpu(cpuidle_coupled_poke_cb, dev->cpu); in cpuidle_coupled_register_device()
677 csd->func = cpuidle_coupled_handle_poke; in cpuidle_coupled_register_device()
678 csd->info = (void *)(unsigned long)dev->cpu; in cpuidle_coupled_register_device()
684 * cpuidle_coupled_unregister_device - unregister a coupled cpuidle device
685 * @dev: struct cpuidle_device for the current cpu
687 * Called from cpuidle_unregister_device to tear down coupled idle. Removes the
688 * cpu from the coupled idle set, and frees the cpuidle_coupled_info struct if
689 * this was the last cpu in the set.
693 struct cpuidle_coupled *coupled = dev->coupled; in cpuidle_coupled_unregister_device()
695 if (cpumask_empty(&dev->coupled_cpus)) in cpuidle_coupled_unregister_device()
698 if (--coupled->refcnt) in cpuidle_coupled_unregister_device()
700 dev->coupled = NULL; in cpuidle_coupled_unregister_device()
704 * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
705 * @coupled: the struct coupled that contains the cpu that is changing state
708 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
712 int cpu = get_cpu(); in cpuidle_coupled_prevent_idle() local
715 coupled->prevent++; in cpuidle_coupled_prevent_idle()
716 cpuidle_coupled_poke_others(cpu, coupled); in cpuidle_coupled_prevent_idle()
723 * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
724 * @coupled: the struct coupled that contains the cpu that is changing state
727 * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
731 int cpu = get_cpu(); in cpuidle_coupled_allow_idle() local
738 coupled->prevent--; in cpuidle_coupled_allow_idle()
740 cpuidle_coupled_poke_others(cpu, coupled); in cpuidle_coupled_allow_idle()
744 static int coupled_cpu_online(unsigned int cpu) in coupled_cpu_online() argument
750 dev = per_cpu(cpuidle_devices, cpu); in coupled_cpu_online()
751 if (dev && dev->coupled) { in coupled_cpu_online()
752 cpuidle_coupled_update_online_cpus(dev->coupled); in coupled_cpu_online()
753 cpuidle_coupled_allow_idle(dev->coupled); in coupled_cpu_online()
760 static int coupled_cpu_up_prepare(unsigned int cpu) in coupled_cpu_up_prepare() argument
766 dev = per_cpu(cpuidle_devices, cpu); in coupled_cpu_up_prepare()
767 if (dev && dev->coupled) in coupled_cpu_up_prepare()
768 cpuidle_coupled_prevent_idle(dev->coupled); in coupled_cpu_up_prepare()