Lines Matching +full:step +full:- +full:down

35 #include <linux/percpu-rwsem.h>
47 * struct cpuhp_cpu_state - Per cpu hotplug state storage
57 * @node: Remote CPU node; for multi-instance, do a
59 * @last: For multi-instance rollback, remember how far we got
62 * @done_up: Signal completion to the issuer of the task for cpu-up
63 * @done_down: Signal completion to the issuer of the task for cpu-down
94 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-up", &cpuhp_state_up_map);
96 STATIC_LOCKDEP_MAP_INIT("cpuhp_state-down", &cpuhp_state_down_map);
116 * struct cpuhp_step - Hotplug state machine step
117 * @name: Name of the step
118 * @startup: Startup function of the step
119 * @teardown: Teardown function of the step
120 * @cant_stop: Bringup/teardown can't be stopped at this step
150 static bool cpuhp_step_empty(bool bringup, struct cpuhp_step *step) in cpuhp_step_empty() argument
152 return bringup ? !step->startup.single : !step->teardown.single; in cpuhp_step_empty()
156 * cpuhp_invoke_callback - Invoke the callbacks for a given state
160 * @node: For multi-instance, do a single entry callback for install/remove
161 * @lastp: For multi-instance rollback, remember how far we got
172 struct cpuhp_step *step = cpuhp_get_step(state); in cpuhp_invoke_callback() local
177 if (st->fail == state) { in cpuhp_invoke_callback()
178 st->fail = CPUHP_INVALID; in cpuhp_invoke_callback()
179 return -EAGAIN; in cpuhp_invoke_callback()
182 if (cpuhp_step_empty(bringup, step)) { in cpuhp_invoke_callback()
187 if (!step->multi_instance) { in cpuhp_invoke_callback()
189 cb = bringup ? step->startup.single : step->teardown.single; in cpuhp_invoke_callback()
191 trace_cpuhp_enter(cpu, st->target, state, cb); in cpuhp_invoke_callback()
193 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
196 cbm = bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
201 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
203 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
209 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
213 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
215 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
230 cbm = !bringup ? step->startup.multi : step->teardown.multi; in cpuhp_invoke_callback()
234 hlist_for_each(node, &step->list) { in cpuhp_invoke_callback()
235 if (!cnt--) in cpuhp_invoke_callback()
238 trace_cpuhp_multi_enter(cpu, st->target, state, cbm, node); in cpuhp_invoke_callback()
240 trace_cpuhp_exit(cpu, st->state, state, ret); in cpuhp_invoke_callback()
261 struct completion *done = bringup ? &st->done_up : &st->done_down; in wait_for_ap_thread()
267 struct completion *done = bringup ? &st->done_up : &st->done_down; in complete_ap_thread()
299 * If set, cpu_up and cpu_down will return -EBUSY and do nothing.
386 cpu_hotplug_disabled--; in __cpu_hotplug_enable()
410 * Architectures that need SMT-specific errata handling during SMT hotplug
480 enum cpuhp_state prev_state = st->state; in cpuhp_set_state()
481 bool bringup = st->state < target; in cpuhp_set_state()
483 st->rollback = false; in cpuhp_set_state()
484 st->last = NULL; in cpuhp_set_state()
486 st->target = target; in cpuhp_set_state()
487 st->single = false; in cpuhp_set_state()
488 st->bringup = bringup; in cpuhp_set_state()
499 bool bringup = !st->bringup; in cpuhp_reset_state()
501 st->target = prev_state; in cpuhp_reset_state()
507 if (st->rollback) in cpuhp_reset_state()
510 st->rollback = true; in cpuhp_reset_state()
513 * If we have st->last we need to undo partial multi_instance of this in cpuhp_reset_state()
516 if (!st->last) { in cpuhp_reset_state()
517 if (st->bringup) in cpuhp_reset_state()
518 st->state--; in cpuhp_reset_state()
520 st->state++; in cpuhp_reset_state()
523 st->bringup = bringup; in cpuhp_reset_state()
531 if (!st->single && st->state == st->target) in __cpuhp_kick_ap()
534 st->result = 0; in __cpuhp_kick_ap()
540 st->should_run = true; in __cpuhp_kick_ap()
541 wake_up_process(st->thread); in __cpuhp_kick_ap()
542 wait_for_ap_thread(st, st->bringup); in __cpuhp_kick_ap()
553 if ((ret = st->result)) { in cpuhp_kick_ap()
568 return -ECANCELED; in bringup_wait_for_ap()
571 kthread_unpark(st->thread); in bringup_wait_for_ap()
581 return -ECANCELED; in bringup_wait_for_ap()
583 if (st->target <= CPUHP_AP_ONLINE_IDLE) in bringup_wait_for_ap()
586 return cpuhp_kick_ap(cpu, st, st->target); in bringup_wait_for_ap()
607 /* Arch-specific enabling code. */ in bringup_cpu()
618 struct mm_struct *mm = idle->active_mm; in finish_cpu()
625 idle->active_mm = &init_mm; in finish_cpu()
638 * st->state will be modified ahead of time, to match state_to_run, as if it
648 if (st->state >= target) in cpuhp_next_state()
651 *state_to_run = ++st->state; in cpuhp_next_state()
653 if (st->state <= target) in cpuhp_next_state()
656 *state_to_run = st->state--; in cpuhp_next_state()
688 * When CPU hotplug is disabled, then taking the CPU down is not in can_rollback_cpu()
694 return st->state <= CPUHP_BRINGUP_CPU; in can_rollback_cpu()
700 enum cpuhp_state prev_state = st->state; in cpuhp_up_callbacks()
706 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_up_callbacks()
707 st->state); in cpuhp_up_callbacks()
724 return st->should_run; in cpuhp_should_run()
735 * - single: runs st->cb_state
736 * - up: runs ++st->state, while st->state < st->target
737 * - down: runs st->state--, while st->state > st->target
744 bool bringup = st->bringup; in cpuhp_thread_fun()
747 if (WARN_ON_ONCE(!st->should_run)) in cpuhp_thread_fun()
751 * ACQUIRE for the cpuhp_should_run() load of ->should_run. Ensures in cpuhp_thread_fun()
752 * that if we see ->should_run we also see the rest of the state. in cpuhp_thread_fun()
764 if (st->single) { in cpuhp_thread_fun()
765 state = st->cb_state; in cpuhp_thread_fun()
766 st->should_run = false; in cpuhp_thread_fun()
768 st->should_run = cpuhp_next_state(bringup, &state, st, st->target); in cpuhp_thread_fun()
769 if (!st->should_run) in cpuhp_thread_fun()
777 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
783 WARN_ON_ONCE(st->result); in cpuhp_thread_fun()
785 st->result = cpuhp_invoke_callback(cpu, state, bringup, st->node, &st->last); in cpuhp_thread_fun()
788 if (st->result) { in cpuhp_thread_fun()
794 WARN_ON_ONCE(st->rollback); in cpuhp_thread_fun()
795 st->should_run = false; in cpuhp_thread_fun()
802 if (!st->should_run) in cpuhp_thread_fun()
827 if (!st->thread) in cpuhp_invoke_ap_callback()
830 st->rollback = false; in cpuhp_invoke_ap_callback()
831 st->last = NULL; in cpuhp_invoke_ap_callback()
833 st->node = node; in cpuhp_invoke_ap_callback()
834 st->bringup = bringup; in cpuhp_invoke_ap_callback()
835 st->cb_state = state; in cpuhp_invoke_ap_callback()
836 st->single = true; in cpuhp_invoke_ap_callback()
843 if ((ret = st->result) && st->last) { in cpuhp_invoke_ap_callback()
844 st->rollback = true; in cpuhp_invoke_ap_callback()
845 st->bringup = !bringup; in cpuhp_invoke_ap_callback()
854 st->node = st->last = NULL; in cpuhp_invoke_ap_callback()
861 enum cpuhp_state prev_state = st->state; in cpuhp_kick_ap_work()
870 trace_cpuhp_enter(cpu, st->target, prev_state, cpuhp_kick_ap_work); in cpuhp_kick_ap_work()
871 ret = cpuhp_kick_ap(cpu, st, st->target); in cpuhp_kick_ap_work()
872 trace_cpuhp_exit(cpu, st->state, prev_state, ret); in cpuhp_kick_ap_work()
892 init_completion(&st->done_up); in cpuhp_init_state()
893 init_completion(&st->done_down); in cpuhp_init_state()
956 * clear_tasks_mm_cpumask - Safely clear tasks' mm_cpumask for a CPU
961 * trivial, there are various non-obvious corner cases, which this function
972 * This function is called after the cpu is taken down and marked in clear_tasks_mm_cpumask()
974 * their mm mask. -- Peter Zijlstra in clear_tasks_mm_cpumask()
976 * full-fledged tasklist_lock. in clear_tasks_mm_cpumask()
990 arch_clear_mm_cpumask_cpu(cpu, t->mm); in clear_tasks_mm_cpumask()
996 /* Take this CPU down. */
1000 enum cpuhp_state target = max((int)st->target, CPUHP_AP_OFFLINE); in take_cpu_down()
1011 * down, that the current state is CPUHP_TEARDOWN_CPU - 1. in take_cpu_down()
1013 WARN_ON(st->state != (CPUHP_TEARDOWN_CPU - 1)); in take_cpu_down()
1038 kthread_park(st->thread); in takedown_cpu()
1054 kthread_unpark(st->thread); in takedown_cpu()
1067 BUG_ON(st->state != CPUHP_AP_IDLE_DEAD); in takedown_cpu()
1092 BUG_ON(st->state != CPUHP_AP_OFFLINE); in cpuhp_report_idle_dead()
1094 st->state = CPUHP_AP_IDLE_DEAD; in cpuhp_report_idle_dead()
1106 enum cpuhp_state prev_state = st->state; in cpuhp_down_callbacks()
1111 pr_debug("CPU DOWN failed (%d) CPU %u state %s (%d)\n", in cpuhp_down_callbacks()
1112 ret, cpu, cpuhp_get_step(st->state)->name, in cpuhp_down_callbacks()
1113 st->state); in cpuhp_down_callbacks()
1117 if (st->state < prev_state) in cpuhp_down_callbacks()
1133 return -EBUSY; in _cpu_down()
1136 return -EINVAL; in _cpu_down()
1147 if (st->state > CPUHP_TEARDOWN_CPU) { in _cpu_down()
1148 st->target = max((int)target, CPUHP_TEARDOWN_CPU); in _cpu_down()
1161 if (st->state > CPUHP_TEARDOWN_CPU) in _cpu_down()
1164 st->target = target; in _cpu_down()
1167 * The AP brought itself down to CPUHP_TEARDOWN_CPU. So we need in _cpu_down()
1171 if (ret && st->state < prev_state) { in _cpu_down()
1172 if (st->state == CPUHP_TEARDOWN_CPU) { in _cpu_down()
1199 return -EOPNOTSUPP; in cpu_down_maps_locked()
1201 return -EBUSY; in cpu_down_maps_locked()
1216 * cpu_device_down - Bring down a cpu device
1227 return cpu_down(dev->id, CPUHP_OFFLINE); in cpu_device_down()
1263 pr_err("Failed to offline CPU%d - error=%d", in smp_shutdown_nonboot_cpus()
1289 * notify_cpu_starting(cpu) - Invoke the callbacks on the starting CPU
1298 enum cpuhp_state target = min((int)st->target, CPUHP_AP_ONLINE); in notify_cpu_starting()
1330 st->state = CPUHP_AP_ONLINE_IDLE; in cpuhp_online_idle()
1344 ret = -EINVAL; in _cpu_up()
1352 if (st->state >= target) in _cpu_up()
1355 if (st->state == CPUHP_OFFLINE) { in _cpu_up()
1371 if (st->state > CPUHP_BRINGUP_CPU) { in _cpu_up()
1400 pr_err("can't online cpu %d because it is not configured as may-hotadd at boot time\n", in cpu_up()
1405 return -EINVAL; in cpu_up()
1415 err = -EBUSY; in cpu_up()
1419 err = -EPERM; in cpu_up()
1430 * cpu_device_up - Bring up a cpu device
1441 return cpu_up(dev->id, CPUHP_ONLINE); in cpu_device_up()
1457 * bringup_hibernate_cpu - Bring up the CPU that we hibernated on
1474 pr_err("Failed to bring hibernate-CPU up!\n"); in bringup_hibernate_cpu()
1501 if (primary == -1) { in freeze_secondary_cpus()
1511 * We take down all of the non-boot CPUs in one shot to avoid races in freeze_secondary_cpus()
1516 pr_info("Disabling non-boot CPUs ...\n"); in freeze_secondary_cpus()
1523 error = -EBUSY; in freeze_secondary_cpus()
1533 pr_err("Error taking CPU%d down: %d\n", cpu, error); in freeze_secondary_cpus()
1541 pr_err("Non-boot CPUs are not disabled\n"); in freeze_secondary_cpus()
1572 pr_info("Enabling non-boot CPUs ...\n"); in thaw_secondary_cpus()
1597 return -ENOMEM; in alloc_frozen_cpus()
1710 * On the tear-down path, timers_dead_cpu() must be invoked
1838 return -EINVAL; in cpuhp_cb_check()
1850 struct cpuhp_step *step; in cpuhp_reserve_state() local
1854 step = cpuhp_hp_states + CPUHP_AP_ONLINE_DYN; in cpuhp_reserve_state()
1858 step = cpuhp_hp_states + CPUHP_BP_PREPARE_DYN; in cpuhp_reserve_state()
1862 return -EINVAL; in cpuhp_reserve_state()
1865 for (i = state; i <= end; i++, step++) { in cpuhp_reserve_state()
1866 if (!step->name) in cpuhp_reserve_state()
1870 return -ENOSPC; in cpuhp_reserve_state()
1899 if (name && sp->name) in cpuhp_store_callbacks()
1900 return -EBUSY; in cpuhp_store_callbacks()
1902 sp->startup.single = startup; in cpuhp_store_callbacks()
1903 sp->teardown.single = teardown; in cpuhp_store_callbacks()
1904 sp->name = name; in cpuhp_store_callbacks()
1905 sp->multi_instance = multi_instance; in cpuhp_store_callbacks()
1906 INIT_HLIST_HEAD(&sp->list); in cpuhp_store_callbacks()
1912 return cpuhp_get_step(state)->teardown.single; in cpuhp_get_teardown_cb()
1916 * Call the startup/teardown function for a step either on the AP or
1960 int cpustate = st->state; in cpuhp_rollback_install()
1982 if (sp->multi_instance == false) in __cpuhp_state_add_instance_cpuslocked()
1983 return -EINVAL; in __cpuhp_state_add_instance_cpuslocked()
1987 if (!invoke || !sp->startup.multi) in __cpuhp_state_add_instance_cpuslocked()
1996 int cpustate = st->state; in __cpuhp_state_add_instance_cpuslocked()
2003 if (sp->teardown.multi) in __cpuhp_state_add_instance_cpuslocked()
2010 hlist_add_head(node, &sp->list); in __cpuhp_state_add_instance_cpuslocked()
2029 * __cpuhp_setup_state_cpuslocked - Setup the callbacks for an hotplug machine state
2031 * @name: Name of the step
2058 return -EINVAL; in __cpuhp_setup_state_cpuslocked()
2080 int cpustate = st->state; in __cpuhp_setup_state_cpuslocked()
2129 if (!sp->multi_instance) in __cpuhp_state_remove_instance()
2130 return -EINVAL; in __cpuhp_state_remove_instance()
2144 int cpustate = st->state; in __cpuhp_state_remove_instance()
2160 * __cpuhp_remove_state_cpuslocked - Remove the callbacks for an hotplug machine state
2179 if (sp->multi_instance) { in __cpuhp_remove_state_cpuslocked()
2180 WARN(!hlist_empty(&sp->list), in __cpuhp_remove_state_cpuslocked()
2196 int cpustate = st->state; in __cpuhp_remove_state_cpuslocked()
2220 dev->offline = true; in cpuhp_offline_cpu_device()
2222 kobject_uevent(&dev->kobj, KOBJ_OFFLINE); in cpuhp_offline_cpu_device()
2229 dev->offline = false; in cpuhp_online_cpu_device()
2231 kobject_uevent(&dev->kobj, KOBJ_ONLINE); in cpuhp_online_cpu_device()
2291 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in state_show()
2293 return sprintf(buf, "%d\n", st->state); in state_show()
2300 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_store()
2310 return -EINVAL; in target_store()
2313 return -EINVAL; in target_store()
2322 ret = !sp->name || sp->cant_stop ? -EINVAL : 0; in target_store()
2327 if (st->state < target) in target_store()
2328 ret = cpu_up(dev->id, target); in target_store()
2330 ret = cpu_down(dev->id, target); in target_store()
2339 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in target_show()
2341 return sprintf(buf, "%d\n", st->target); in target_show()
2348 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_store()
2357 st->fail = fail; in fail_store()
2362 return -EINVAL; in fail_store()
2368 return -EINVAL; in fail_store()
2376 if (fail <= CPUHP_BRINGUP_CPU && st->state > CPUHP_BRINGUP_CPU) in fail_store()
2377 return -EINVAL; in fail_store()
2384 if (!sp->startup.single && !sp->teardown.single) in fail_store()
2385 ret = -EINVAL; in fail_store()
2390 st->fail = fail; in fail_store()
2398 struct cpuhp_cpu_state *st = per_cpu_ptr(&cpuhp_state, dev->id); in fail_show()
2400 return sprintf(buf, "%d\n", st->fail); in fail_show()
2428 if (sp->name) { in states_show()
2429 cur = sprintf(buf, "%3d: %s\n", i, sp->name); in states_show()
2465 return -EINVAL; in __store_smt_control()
2468 return -EPERM; in __store_smt_control()
2471 return -ENODEV; in __store_smt_control()
2498 return -ENODEV; in __store_smt_control()
2515 return snprintf(buf, PAGE_SIZE - 2, "%s\n", state); in control_show()
2528 return snprintf(buf, PAGE_SIZE - 2, "%d\n", sched_smt_active()); in active_show()
2546 return sysfs_create_group(&cpu_subsys.dev_root->kobj, in cpu_smt_sysfs_init()
2558 ret = sysfs_create_group(&cpu_subsys.dev_root->kobj, in cpuhp_sysfs_init()
2568 ret = sysfs_create_group(&dev->kobj, &cpuhp_cpu_attr_group); in cpuhp_sysfs_init()
2585 /* cpu_bit_bitmap[0] is empty - so we can back into it */
2648 * IPI/NMI broadcasts when shutting down CPUs. Invocation from in set_cpu_online()