3  * coupled.c - helper functions to enter the same idle state on multiple cpus
24  * cpus cannot be independently powered down, either due to
31  * shared between the cpus (L2 cache, interrupt controller, and
33  * be tightly controlled on both cpus.
36  * WFI state until all cpus are ready to enter a coupled state, at
38  * cpus at approximately the same time.
40  * Once all cpus are ready to enter idle, they are woken by an smp
42  * cpus will find work to do, and choose not to enter idle.  A
43  * final pass is needed to guarantee that all cpus will call the
46  * ready counter matches the number of online coupled cpus.  If any
47  * cpu exits idle, the other cpus will decrement their counter and
56  * and only read after all the cpus are ready for the coupled idle
60  * of cpus in the coupled set that are currently or soon will be
61  * online.  waiting_count tracks the number of cpus that are in
63  * ready_count tracks the number of cpus that are in the ready loop
69  *    coupled cpus, usually the same as cpu_possible_mask if all cpus
77  *    state that affects multiple cpus.
80  *    that affects multiple cpus.  This function is guaranteed to be
81  *    called on all cpus at approximately the same time.  The driver
82  *    should ensure that the cpus all abort together if any cpu tries
88  * struct cpuidle_coupled - data for set of cpus that share a coupled idle state
89  * @coupled_cpus: mask of cpus that are part of the coupled set
90  * @requested_state: array of requested states for cpus in the coupled set
91  * @ready_waiting_counts: combined count of cpus  in ready or waiting loops
92  * @online_count: count of cpus that are online
118  * in use.  This prevents a deadlock where two cpus are waiting for each others
131  * cpuidle_coupled_parallel_barrier - synchronize all online coupled cpus
136  * cpus in the same coupled group have called this function.  Once any caller
172  * Returns true if the target state is coupled with cpus besides this one
215  * is equal to the number of online cpus.  Prevents a race where one cpu
218  * down from the number of online cpus without going through the coupled idle
222  * counter was equal to the number of online cpus.
238  * cpuidle_coupled_no_cpus_ready - check if no cpus in a coupled set are ready
241  * Returns true if all of the cpus in a coupled set are out of the ready loop.
250  * cpuidle_coupled_cpus_ready - check if all cpus in a coupled set are ready
253  * Returns true if all cpus coupled to this target state are in the ready loop
262  * cpuidle_coupled_cpus_waiting - check if all cpus in a coupled set are waiting
265  * Returns true if all cpus coupled to this target state are in the wait loop
274  * cpuidle_coupled_no_cpus_waiting - check if no cpus in coupled set are waiting
277  * Returns true if all of the cpus in a coupled set are out of the waiting loop.
290  * Returns the deepest idle state that all coupled cpus can enter
340  * cpuidle_coupled_poke_others - wake up all other cpus that may be waiting
344  * Calls cpuidle_coupled_poke on all other online cpus.
363  * Returns the number of waiting cpus.
390 	 * cpus will increment ready_count and then spin until they  in cpuidle_coupled_set_not_waiting()
441 	cpumask_t cpus;  in cpuidle_coupled_any_pokes_pending()  local
444 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_any_pokes_pending()
445 	ret = cpumask_and(&cpus, &cpuidle_coupled_poke_pending, &cpus);  in cpuidle_coupled_any_pokes_pending()
451  * cpuidle_enter_state_coupled - attempt to enter a state with coupled cpus
456  * Coordinate with coupled cpus to enter the target state.  This is a two
457  * stage process.  In the first stage, the cpus are operating independently,
459  * To save as much power as possible, the first cpus to call this function will
461  * all the other cpus to call this function.  Once all coupled cpus are idle,
462  * the second stage will start.  Each coupled cpu will spin until all cpus have
499 	 * all the other cpus out of their waiting state so they can  in cpuidle_enter_state_coupled()
500 	 * enter a deeper state.  This can race with one of the cpus  in cpuidle_enter_state_coupled()
511 	 * Wait for all coupled cpus to be idle, using the deepest state  in cpuidle_enter_state_coupled()
514 	 * two cpus could arrive at the waiting loop at the same time,  in cpuidle_enter_state_coupled()
551 	 * All coupled cpus are probably idle.  There is a small chance that  in cpuidle_enter_state_coupled()
552 	 * one of the other cpus just became active.  Increment the ready count,  in cpuidle_enter_state_coupled()
553 	 * and spin until all coupled cpus have incremented the counter. Once a  in cpuidle_enter_state_coupled()
555 	 * spin until either all cpus have incremented the ready counter, or  in cpuidle_enter_state_coupled()
561 		/* Check if any other cpus bailed out of idle. */  in cpuidle_enter_state_coupled()
570 	 * Make sure read of all cpus ready is done before reading pending pokes  in cpuidle_enter_state_coupled()
576 	 * cpu saw that all cpus were waiting.  The cpu that reentered idle will  in cpuidle_enter_state_coupled()
582 	 * coupled idle state of all cpus and retry.  in cpuidle_enter_state_coupled()
586 		/* Wait for all cpus to see the pending pokes */  in cpuidle_enter_state_coupled()
591 	/* all cpus have acked the coupled state */  in cpuidle_enter_state_coupled()
604 	 * other cpus will need to spin waiting for the cpu that is processing  in cpuidle_enter_state_coupled()
606 	 * all other cpus will loop back into the safe idle state instead of  in cpuidle_enter_state_coupled()
616 	 * Wait until all coupled cpus have exited idle.  There is no risk that  in cpuidle_enter_state_coupled()
628 	cpumask_t cpus;  in cpuidle_coupled_update_online_cpus()  local
629 	cpumask_and(&cpus, cpu_online_mask, &coupled->coupled_cpus);  in cpuidle_coupled_update_online_cpus()
630 	coupled->online_count = cpumask_weight(&cpus);  in cpuidle_coupled_update_online_cpus()
638  * cpuidle_coupled struct for this set of coupled cpus, or creates one if none
703  * cpuidle_coupled_prevent_idle - prevent cpus from entering a coupled state
706  * Disables coupled cpuidle on a coupled set of cpus.  Used to ensure that
707  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
713 	/* Force all cpus out of the waiting loop. */  in cpuidle_coupled_prevent_idle()
722  * cpuidle_coupled_allow_idle - allows cpus to enter a coupled state
725  * Enables coupled cpuidle on a coupled set of cpus.  Used to ensure that
726  * cpu_online_mask doesn't change while cpus are coordinating coupled idle.
738 	/* Force cpus out of the prevent loop. */  in cpuidle_coupled_allow_idle()